From ca0f32641949b93b80d846337a58b7d475ea1898 Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Fri, 22 Sep 2023 20:22:17 -0400 Subject: [PATCH 01/54] Update testing-needed job to not fail when testing-done-e2e-full label presents (#227) This patch updates the check-label job to not fail the PR if it contains a `testing-done-e2e-full` label. This is to address the case where a PR may require `testing-needed-e2e-fast`, but developers have conducted a full e2e validation and attached the `testing-done-e2e-full` label. --- .github/workflows/testing-needed.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/testing-needed.yml b/.github/workflows/testing-needed.yml index e18a3d9d6..1f7419a2d 100644 --- a/.github/workflows/testing-needed.yml +++ b/.github/workflows/testing-needed.yml @@ -18,8 +18,8 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - # The pull_request_target event runs in the context of the BASE of the pull request. - # We need to checkout the HEAD of the pull request to be able to check the diff. + # The pull_request_target event runs in the context of the PR's BASE. + # We need to checkout the PR's HEAD to be able to check the diff. ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - name: Check diff @@ -33,7 +33,8 @@ jobs: uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }} - # TODO: Add testing-needed-e2e-full label once the pipeline supports running e2e-full tests. + # TODO: Add testing-needed-e2e-full label once the internal pipeline + # supports running e2e-full tests with the VM-Operator build from PR. script: | if (${{ steps.check-diff.outputs.lines_of_change > 99 }}) { await github.rest.issues.addLabels({ @@ -81,12 +82,14 @@ jobs: - name: do-not-merge env: LABEL_NAMES: ${{ needs.verify-change.outputs.label_names }} - # the step will run if one of the labels is present and the corresponding + # This step will run if one of the labels is present and the related # label indicating testing is done is not present, ex. the label - # testing-needed-e2e-fast is present without also testing-done-e2e-fast + # testing-needed-e2e-fast is present without also testing-done-e2e-fast. + # Additionally, it should never run if testing-done-e2e-full is present. if: | contains(env.LABEL_NAMES, format('testing-needed-{0}', matrix.test-type)) && - !contains(env.LABEL_NAMES, format('testing-done-{0}', matrix.test-type)) + !contains(env.LABEL_NAMES, format('testing-done-{0}', matrix.test-type)) && + !contains(env.LABEL_NAMES, 'testing-done-e2e-full') run: | echo "Pull request is labeled as 'testing-needed-${{ matrix.test-type }}'" exit 1 From da4884321d0bf1b2ab7b86cd899148a9334011e5 Mon Sep 17 00:00:00 2001 From: Yiyi Zhou <91219164+zyiyi11@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:22:58 -0700 Subject: [PATCH 02/54] :book: Add troubleshooting page for ip assignment issue (#220) * Add troubleshooting page for ip assignment issue --- .../troubleshooting/ip-assignment.md | 133 +++++++++++++++++- 1 file changed, 131 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/troubleshooting/ip-assignment.md b/docs/tutorials/troubleshooting/ip-assignment.md index acd65ed62..20819a31b 100644 --- a/docs/tutorials/troubleshooting/ip-assignment.md +++ b/docs/tutorials/troubleshooting/ip-assignment.md @@ -1,2 +1,131 @@ -# Virtual Machine No valid IP Address -// TODO (github.com/vmware-tanzu/vm-operator#193) +# IP Assignment +This page describes how to troubleshoot when VM was created but was stuck in the status with no valid IP addresses. + +## Procedure +### 1. Access Your Kubernetes Namespace +Ensure you are in the correct Kubernetes context. Use the following command to set the context to the desired namespace. + +```console +$ kubectl config use-context +``` + +See [Get and Use the Supervisor Context](https://docs.vmware.com/en/VMware-vSphere/8.0/vsphere-with-tanzu-services-workloads/GUID-63A1C273-DC75-420B-B7FD-47CB25A50A2C.html#GUID-63A1C273-DC75-420B-B7FD-47CB25A50A2C) if you need help accessing Supervisor clusters. + + +### 2. Verify VM Network Settings +Check if the VM's network settings match the underlying networking infrastructure. If you specify an nsx-t network in a vds networking environment (or vice versa), you may encounter an error message. + +Use the following command to check the VM's network settings: +```console +$ kubectl describe vm -n +``` + +The output is similar to the following: +```console +Spec: + Network Interfaces: + Network Type: nsx-t +... +Events: +Type Reason Age From Message +---- ------ ---- ---- ------- +Warning CreateOrUpdateFailure 5s (x16 over 2m24s) vmware-system-vmop/vmware-system-vmop-controller-manager-5ff5d769d8-6rwqc/virtualmachine-controller no matches for kind "VirtualNetworkInterface" in version "vmware.com/v1alpha1" +``` + +**Fix**: Not specify network interface during VM deployment, VM Operator will utilize default networking. + +### 3. Check NCP VirtualNetworkInterface Status (NSX-T Networking) +For NSX-T networking, verify the status of the VirtualNetworkInterface. The expected conditions type should be "Ready," and the IP Addresses should return valid addresses. + +**Note** if `vm.spec.networkInterfaces[0].networkName` is empty, then `vnetif_name` should be `-lsp`. Otherwise, `vnetif_name` should be `--lsp`. + +Use the following command to check the `VirtualNetworkInterface` status: +```console +$ kubectl describe virtualnetworkinterfaces -n +``` + +The output is similar to the following: +```console +Status: + Conditions: + Status: True + Type: Ready + Ip Addresses: + Gateway: 172.26.0.33 + Ip: 172.26.0.34 + Subnet Mask: 255.255.255.240 +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRealizeNSXResource 25m nsx-container-ncp Successfully realized NSX resource for VirtualNetworkInterface +``` + +**Fix** Contact your VI Admin to verify the networking health status. + +### 4. Check NetworkInterfacesStatus (VDS Networking) +For VDS networking, inspect the NetworkInterface status. The conditions type should be "Ready," and the IP Configs should return valid addresses. + +Use the following command to check the `NetworkInterface` status: +```console +$ kubectl describe networkinterface -n +``` + +The output is similar to the following: +```console +Status: + Conditions: + Last Transition Time: 2023-09-18T19:17:38Z + Status: True + Type: Ready + Ip Configs: + Gateway: 192.168.1.1 + Ip: 192.168.128.42 + Ip Family: IPv4 + Subnet Mask: 255.255.0.0 + Network ID: dvportgroup-55 +Events: +``` + +**Fix** Contact your VI Admin to verify the networking health status. + +### 5. Bootstrap +When network interfaces issues are ruled out, we will troubleshoot issues caused by [Bootstrap Providers](https://vm-operator.readthedocs.io/en/stable/concepts/workloads/guest/). Here, we'll explore troubleshooting steps for **CloudInit**, **Sysprep**, and **vAppConfig** issues that may affect network connectivity. +#### a. CloudInit +For VM deployed using CloudInit bootstrap, if the VM is powered on but doesn't have a valid IPV4 IP assigned, it usually indicates that the CloudInit failed. Follow the steps below to troubleshoot: + +1. Check `GuestCustomization` condition in VM: When `GuestCustomization` condition shows false, it indicates GOSC or CloudInit failure. + - *Alternative* - Check Customization Reconfigure Event: In the vCenter UI, verify if the `Customization Reconfigure` event is present in the VM's events. Its absence suggests a CloudInit failure. + +2. Inspect VM ExtraConfig Values: + - Ensure that ExtraConfig[guestinfo.metadata] contains metadata generated by the vm-operator, including network configurations and hostname. + - Confirm that ExtraConfig[guestinfo.userdata] contains the user-supplied cloud-config data. + +3. Examine Cloud-Init Logs: Log in to the virtual machine using the web console. Access the VM's filesystem and locate the Cloud-Init logs at `/var/log/cloud-init.log` and `/var/log/cloud-init-output.log`. + +#### b. Sysprep +For VM deployed using Sysprep bootstrap, if the VM is powered on but doesn't have a valid IPV4 IP assigned, it usually indicates that the GOSC failed. Follow the steps below to troubleshoot: + +1. Check `GuestCustomization` condition in VM: When `GuestCustomization` condition shows false, it indicates GOSC or Sysprep failure. + - *Alternative* - Check Customization Succeeded Event: In the vCenter UI, verify if the `Customization of VM succeeded` event is present in the VM's events. Its absence indicates GOSC or Sysprep failure. + +2. Inspect GOSC Status: Log in to the virtual machine using the web console. Check the log file at `C:/Windows/TEMP/vmware-imc/guestcust` (the path may vary based on the Windows version) to confirm GOSC status. + +3. Validate Sysprep Answer File: Inside the VM, ensure all templating expressions have been parsed correctly. For example, you should see `{{ V1alpha1_FirstNicMacAddr }}` converted to `00-11-22-33-aa-bb-cc`. +The Sysprep content file should be located at `C:\sysprep1001\sysprep.xml` inside the VM. + +4. Check the GOSC and Sysprep Logs: Examine logs at the following paths within the VM for more details: +``` +C:/Windows/Panther/setuperr +C:/Windows/Panther/Unattendgc/setuperr +C:/Windows/System32/Sysprep/Panther/setuperr +``` + +#### c. vAppConfig +For VM deployed using vAppConfig bootstrap, if the VM is powered on but doesn't have a valid IPV4 IP assigned, it usually indicates that the GOSC or vAppConfig failed. Follow the steps below to troubleshoot: + +1. Check `GuestCustomization` condition in VM: When `GuestCustomization` condition shows false, it indicates GOSC or vAppConfig failure. + - *Alternative* - Check Customization Succeeded Event: In the vCenter UI, ensure that the `Customization of VM succeeded` event is present in the VM's events. Its absence indicates GOSC or vAppConfig failure. + +2. Verify VM `VAppPropertyInfo`: Inspect the `config.vAppConfig.property` of the VM to ensure all templating expressions have been parsed correctly. + +3. Inspect Logs: Log in to the virtual machine using the web console. Check the log file `/var/log/vmware-imc/toolsDeployPkg.log` file look for string `Executing Traditional GOSC workflow`. \ No newline at end of file From 4298ad0277b3ac51fe969da179860ba7b812d907 Mon Sep 17 00:00:00 2001 From: akutz Date: Mon, 25 Sep 2023 16:17:01 -0500 Subject: [PATCH 03/54] Update GoVmomi to 8.0U2 This patch updates the GoVmomi dependency to 8.0U2. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e3993b1d2..b92fdd4f7 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/vmware-tanzu/vm-operator/api v0.0.0-00010101000000-000000000000 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-00010101000000-000000000000 github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-00010101000000-000000000000 - github.com/vmware/govmomi v0.28.1-0.20230918130735-a83ec3f76646 + github.com/vmware/govmomi v0.31.0 // per the following dependabot alerts: // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/24 golang.org/x/net v0.13.0 // indirect diff --git a/go.sum b/go.sum index b36422310..610b42d4d 100644 --- a/go.sum +++ b/go.sum @@ -455,8 +455,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vmware-tanzu/image-registry-operator-api v0.0.0-20230526154708-f67dac7c805f h1:qpQD1XWbDpti3fBxKfQq5YRPmdQkbNS36RynprKJKoc= github.com/vmware-tanzu/image-registry-operator-api v0.0.0-20230526154708-f67dac7c805f/go.mod h1:S0HMBgdo3S/0a5hwq+Ya4XZI2aEDtGkSGeojU1cINOg= -github.com/vmware/govmomi v0.28.1-0.20230918130735-a83ec3f76646 h1:bLvbB/J35Wp4TCRdfHAPTtrQCEvfDOCo65EkzvbZ2cE= -github.com/vmware/govmomi v0.28.1-0.20230918130735-a83ec3f76646/go.mod h1:JA63Pg0SgQcSjk+LuPzjh3rJdcWBo/ZNCIwbb1qf2/0= +github.com/vmware/govmomi v0.31.0 h1:+NC7le8yeXj7f4YUC841jgdWsehN7A3ivqLxm79eKyo= +github.com/vmware/govmomi v0.31.0/go.mod h1:JA63Pg0SgQcSjk+LuPzjh3rJdcWBo/ZNCIwbb1qf2/0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From 9280cde411d58209a1ebfe89898a5758b5cfc14d Mon Sep 17 00:00:00 2001 From: Arunesh Pandey Date: Mon, 25 Sep 2023 15:12:07 -0700 Subject: [PATCH 04/54] Bump up min Go version to 1.20 and other dependent changes (#228) controller-runtime v15 has a min Golang version requirement that breaks the container builds. This change bumps up the Go version used to build the container. Starting 1.21 Golang, go.mod requires the Go version and treats that as min required go version to compile your binary. So, also bump up the Go version for local builds. Additional changes: - As per https://github.com/golang/go/issues/56319 and https://github.com/golang/go/issues/54880, global rand is automatically, randomly seeded. So we don't need to call `Seed` anymore. - Package ioutil has been deprecated. Port the code to use replacement methods. --- .github/workflows/ci.yml | 2 +- Dockerfile | 2 +- go.mod | 2 +- main.go | 3 --- pkg/util/enc_test.go | 4 ++-- test/builder/util.go | 6 +++--- 6 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a634c5693..86cd93469 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: ci env: - GO_VERSION: 1.19.3 + GO_VERSION: 1.20.0 on: pull_request: diff --git a/Dockerfile b/Dockerfile index 49f2e7ccf..f1cf4274b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Go version used to build the binaries. -ARG GO_VERSION=1.18.4 +ARG GO_VERSION=1.20 ## Docker image used to build the binaries. FROM golang:${GO_VERSION} as builder diff --git a/go.mod b/go.mod index e3993b1d2..f66598201 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/vmware-tanzu/vm-operator -go 1.18 +go 1.20 replace ( github.com/envoyproxy/go-control-plane => github.com/envoyproxy/go-control-plane v0.9.4 diff --git a/main.go b/main.go index 2a12444ce..c40785abc 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ package main import ( "crypto/tls" "flag" - "math/rand" "net/http" "net/http/pprof" "os" @@ -119,8 +118,6 @@ func main() { setupLog.Info("Starting VM Operator controller", "version", pkg.BuildVersion, "buildnumber", pkg.BuildNumber, "buildtype", pkg.BuildType, "commit", pkg.BuildCommit) - rand.Seed(time.Now().UnixNano()) - profilerAddress := flag.String( "profiler-address", defaultProfilerAddr, diff --git a/pkg/util/enc_test.go b/pkg/util/enc_test.go index 938b729c0..6d22f13e5 100644 --- a/pkg/util/enc_test.go +++ b/pkg/util/enc_test.go @@ -7,7 +7,7 @@ import ( "bytes" "compress/gzip" "encoding/base64" - "io/ioutil" + "io" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -170,7 +170,7 @@ var _ = Describe("EncodeGzipBase64", func() { Expect(err).NotTo(HaveOccurred()) defer Expect(gzipReader.Close()).To(Succeed()) - ungzipped, err := ioutil.ReadAll(gzipReader) + ungzipped, err := io.ReadAll(gzipReader) Expect(err).NotTo(HaveOccurred()) Expect(input).Should(Equal(string(ungzipped))) }) diff --git a/test/builder/util.go b/test/builder/util.go index b1ac9ece5..196a1124d 100644 --- a/test/builder/util.go +++ b/test/builder/util.go @@ -12,7 +12,7 @@ import ( "encoding/pem" "fmt" "io" - "io/ioutil" + "os" "path/filepath" "github.com/google/uuid" @@ -604,7 +604,7 @@ func indexOfVersion( func LoadCRDs(rootFilePath string) ([]*apiextensionsv1.CustomResourceDefinition, error) { // Read the CRD files. - files, err := ioutil.ReadDir(rootFilePath) + files, err := os.ReadDir(rootFilePath) if err != nil { return nil, err } @@ -641,7 +641,7 @@ func LoadCRDs(rootFilePath string) ([]*apiextensionsv1.CustomResourceDefinition, // copied from https://github.com/kubernetes-sigs/controller-runtime/blob/5bf44d2ffd6201703508e11fbae74fcedc5ce148/pkg/envtest/crd.go#L434-L458 func readDocuments(fp string) ([][]byte, error) { //nolint:gosec - b, err := ioutil.ReadFile(fp) + b, err := os.ReadFile(fp) if err != nil { return nil, err } From f63362598a7709b10f073e47ba5a743e3a98edac Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Tue, 26 Sep 2023 09:47:37 -0700 Subject: [PATCH 05/54] Include the controller-runtime webhook patches for v1a2 and update v1a1 webhook names (#226) This change - includes the controller-runtime conversion webhook patches for v1a2 CRs and - updates the v1a1 webhook names to include v1alpha1 --- config/crd/kustomization.yaml | 28 +++++++++---------- config/webhook/manifests.yaml | 14 +++++----- .../mutation/virtualmachine_mutator.go | 2 +- .../virtualmachine_mutator_suite_test.go | 2 +- .../validation/virtualmachine_validator.go | 2 +- .../virtualmachine_validator_suite_test.go | 2 +- .../mutation/virtualmachineclass_mutator.go | 2 +- .../virtualmachineclass_mutator_suite_test.go | 2 +- .../virtualmachineclass_validator.go | 2 +- ...irtualmachineclass_validator_suite_test.go | 2 +- .../virtualmachinepublishrequest_validator.go | 2 +- ...hinepublishrequest_validator_suite_test.go | 2 +- .../mutation/virtualmachineservice_mutator.go | 2 +- ...irtualmachineservice_mutator_suite_test.go | 2 +- .../virtualmachineservice_validator.go | 2 +- ...tualmachineservice_validator_suite_test.go | 2 +- ...virtualmachinesetresourcepolicy_mutator.go | 2 +- ...inesetresourcepolicy_mutator_suite_test.go | 2 +- ...rtualmachinesetresourcepolicy_validator.go | 2 +- ...esetresourcepolicy_validator_suite_test.go | 2 +- .../validation/webconsolerequest_validator.go | 2 +- .../webconsolerequest_validator_suite_test.go | 2 +- 22 files changed, 41 insertions(+), 41 deletions(-) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 53d8066ec..707333155 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -25,25 +25,25 @@ patches: patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD -#- patches/webhook_in_clustervirtualmachineimages.yaml -#- patches/webhook_in_virtualmachineclasses.yaml -#- patches/webhook_in_virtualmachineimages.yaml -#- patches/webhook_in_virtualmachinepublishrequests.yaml -#- patches/webhook_in_virtualmachines.yaml -#- patches/webhook_in_virtualmachineservices.yaml -#- patches/webhook_in_virtualmachinesetresourcepolicies.yaml +- patches/webhook_in_clustervirtualmachineimages.yaml +- patches/webhook_in_virtualmachineclasses.yaml +- patches/webhook_in_virtualmachineimages.yaml +- patches/webhook_in_virtualmachinepublishrequests.yaml +- patches/webhook_in_virtualmachines.yaml +- patches/webhook_in_virtualmachineservices.yaml +- patches/webhook_in_virtualmachinesetresourcepolicies.yaml #- patches/webhook_in_virtualmachinewebconsolerequests.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD -#- patches/cainjection_in_clustervirtualmachineimages.yaml -#- patches/cainjection_in_virtualmachineclasses.yaml -#- patches/cainjection_in_virtualmachineimages.yaml -#- patches/cainjection_in_virtualmachinepublishrequests.yaml -#- patches/cainjection_in_virtualmachines.yaml -#- patches/cainjection_in_virtualmachineservices.yaml -#- patches/cainjection_in_virtualmachinesetresourcepolicies.yaml +- patches/cainjection_in_clustervirtualmachineimages.yaml +- patches/cainjection_in_virtualmachineclasses.yaml +- patches/cainjection_in_virtualmachineimages.yaml +- patches/cainjection_in_virtualmachinepublishrequests.yaml +- patches/cainjection_in_virtualmachines.yaml +- patches/cainjection_in_virtualmachineservices.yaml +- patches/cainjection_in_virtualmachinesetresourcepolicies.yaml #- patches/cainjection_in_virtualmachinewebconsolerequests.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index af666c097..faaceb654 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -14,7 +14,7 @@ webhooks: namespace: system path: /default-mutate-vmoperator-vmware-com-v1alpha1-virtualmachine failurePolicy: Fail - name: default.mutating.virtualmachine.vmoperator.vmware.com + name: default.mutating.virtualmachine.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -84,7 +84,7 @@ webhooks: namespace: system path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachine failurePolicy: Fail - name: default.validating.virtualmachine.vmoperator.vmware.com + name: default.validating.virtualmachine.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -105,7 +105,7 @@ webhooks: namespace: system path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineclass failurePolicy: Fail - name: default.validating.virtualmachineclass.vmoperator.vmware.com + name: default.validating.virtualmachineclass.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -126,7 +126,7 @@ webhooks: namespace: system path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinepublishrequest failurePolicy: Fail - name: default.validating.virtualmachinepublishrequest.vmoperator.vmware.com + name: default.validating.virtualmachinepublishrequest.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -147,7 +147,7 @@ webhooks: namespace: system path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineservice failurePolicy: Fail - name: default.validating.virtualmachineservice.vmoperator.vmware.com + name: default.validating.virtualmachineservice.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -168,7 +168,7 @@ webhooks: namespace: system path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy failurePolicy: Fail - name: default.validating.virtualmachinesetresourcepolicy.vmoperator.vmware.com + name: default.validating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -189,7 +189,7 @@ webhooks: namespace: system path: /default-validate-vmoperator-vmware-com-v1alpha1-webconsolerequest failurePolicy: Fail - name: default.validating.webconsolerequest.vmoperator.vmware.com + name: default.validating.webconsolerequest.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com diff --git a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go index 57e369502..82e23f950 100644 --- a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go +++ b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go @@ -35,7 +35,7 @@ const ( defaultNamedNetwork = "VM Network" ) -// +kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha1-virtualmachine,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,verbs=create;update,versions=v1alpha1,name=default.mutating.virtualmachine.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha1-virtualmachine,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,verbs=create;update,versions=v1alpha1,name=default.mutating.virtualmachine.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachine,verbs=get;list // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachine/status,verbs=get // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineimages,verbs=get;list;watch diff --git a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_suite_test.go b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_suite_test.go index 071f6d903..9927a0ccf 100644 --- a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_suite_test.go +++ b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_suite_test.go @@ -17,7 +17,7 @@ import ( var suite = builder.NewTestSuiteForMutatingWebhook( mutation.AddToManager, mutation.NewMutator, - "default.mutating.virtualmachine.vmoperator.vmware.com") + "default.mutating.virtualmachine.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Mutating webhook suite", intgTests, uniTests) diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go index 0e435fd91..6babc8d51 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go @@ -67,7 +67,7 @@ const ( settingAnnotationNotAllowed = "adding this annotation is not allowed" ) -// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachine,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,versions=v1alpha1,name=default.validating.virtualmachine.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachine,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,versions=v1alpha1,name=default.validating.virtualmachine.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachines,verbs=get;list // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachines/status,verbs=get diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_suite_test.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_suite_test.go index 5740f8d49..7a155a15f 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_suite_test.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_suite_test.go @@ -16,7 +16,7 @@ import ( var suite = builder.NewTestSuiteForValidatingWebhook( validation.AddToManager, validation.NewValidator, - "default.validating.virtualmachine.vmoperator.vmware.com") + "default.validating.virtualmachine.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Validation webhook suite", intgTests, unitTests) diff --git a/webhooks/virtualmachineclass/v1alpha1/mutation/virtualmachineclass_mutator.go b/webhooks/virtualmachineclass/v1alpha1/mutation/virtualmachineclass_mutator.go index 371e79fa7..45134b9e4 100644 --- a/webhooks/virtualmachineclass/v1alpha1/mutation/virtualmachineclass_mutator.go +++ b/webhooks/virtualmachineclass/v1alpha1/mutation/virtualmachineclass_mutator.go @@ -25,7 +25,7 @@ const ( webHookName = "default" ) -// -kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha1-virtualmachineclass,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachineclasses,verbs=create;update,versions=v1alpha1,name=default.mutating.virtualmachineclass.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// -kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha1-virtualmachineclass,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachineclasses,verbs=create;update,versions=v1alpha1,name=default.mutating.virtualmachineclass.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // -kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineclass,verbs=get;list // -kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineclass/status,verbs=get diff --git a/webhooks/virtualmachineclass/v1alpha1/mutation/virtualmachineclass_mutator_suite_test.go b/webhooks/virtualmachineclass/v1alpha1/mutation/virtualmachineclass_mutator_suite_test.go index 7285eed5d..df0e8bc83 100644 --- a/webhooks/virtualmachineclass/v1alpha1/mutation/virtualmachineclass_mutator_suite_test.go +++ b/webhooks/virtualmachineclass/v1alpha1/mutation/virtualmachineclass_mutator_suite_test.go @@ -16,7 +16,7 @@ import ( var suite = builder.NewTestSuiteForMutatingWebhook( mutation.AddToManager, mutation.NewMutator, - "default.mutating.virtualmachineclass.vmoperator.vmware.com") + "default.mutating.virtualmachineclass.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Mutating webhook suite", intgTests, uniTests) diff --git a/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator.go b/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator.go index f7ad18fb7..2d815aa79 100644 --- a/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator.go +++ b/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator.go @@ -33,7 +33,7 @@ const ( invalidMemoryReqMsg = "memory request must not be larger than the memory limit" ) -// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineclass,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachineclasses,versions=v1alpha1,name=default.validating.virtualmachineclass.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineclass,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachineclasses,versions=v1alpha1,name=default.validating.virtualmachineclass.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineclasses,verbs=get;list // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineclasses/status,verbs=get diff --git a/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator_suite_test.go b/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator_suite_test.go index a19f3fe75..6213b788d 100644 --- a/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator_suite_test.go +++ b/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator_suite_test.go @@ -16,7 +16,7 @@ import ( var suite = builder.NewTestSuiteForValidatingWebhook( validation.AddToManager, validation.NewValidator, - "default.validating.virtualmachineclass.vmoperator.vmware.com") + "default.validating.virtualmachineclass.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Validation webhook suite", intgTests, unitTests) diff --git a/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator.go b/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator.go index c658d71b2..bf3e65c7b 100644 --- a/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator.go +++ b/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator.go @@ -30,7 +30,7 @@ const ( webHookName = "default" ) -// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinepublishrequest,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachinepublishrequests,versions=v1alpha1,name=default.validating.virtualmachinepublishrequest.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinepublishrequest,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachinepublishrequests,versions=v1alpha1,name=default.validating.virtualmachinepublishrequest.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinepublishrequests,verbs=get;list // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinepublishrequests/status,verbs=get // +kubebuilder:rbac:groups=imageregistry.vmware.com,resources=contentlibraries,verbs=get;list; diff --git a/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator_suite_test.go b/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator_suite_test.go index 12a7e5ee7..acc508bb5 100644 --- a/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator_suite_test.go +++ b/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator_suite_test.go @@ -16,7 +16,7 @@ import ( var suite = builder.NewTestSuiteForValidatingWebhook( validation.AddToManager, validation.NewValidator, - "default.validating.virtualmachinepublishrequest.vmoperator.vmware.com") + "default.validating.virtualmachinepublishrequest.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Validation webhook suite", intgTests, unitTests) diff --git a/webhooks/virtualmachineservice/v1alpha1/mutation/virtualmachineservice_mutator.go b/webhooks/virtualmachineservice/v1alpha1/mutation/virtualmachineservice_mutator.go index e5d7f4d0e..7845dc98f 100644 --- a/webhooks/virtualmachineservice/v1alpha1/mutation/virtualmachineservice_mutator.go +++ b/webhooks/virtualmachineservice/v1alpha1/mutation/virtualmachineservice_mutator.go @@ -24,7 +24,7 @@ const ( webHookName = "default" ) -// -kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha1-virtualmachineservice,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachineservices,verbs=create;update,versions=v1alpha1,name=default.mutating.virtualmachineservice.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// -kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha1-virtualmachineservice,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachineservices,verbs=create;update,versions=v1alpha1,name=default.mutating.virtualmachineservice.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // -kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservice,verbs=get;list // -kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservice/status,verbs=get diff --git a/webhooks/virtualmachineservice/v1alpha1/mutation/virtualmachineservice_mutator_suite_test.go b/webhooks/virtualmachineservice/v1alpha1/mutation/virtualmachineservice_mutator_suite_test.go index f0c392ddc..a027196af 100644 --- a/webhooks/virtualmachineservice/v1alpha1/mutation/virtualmachineservice_mutator_suite_test.go +++ b/webhooks/virtualmachineservice/v1alpha1/mutation/virtualmachineservice_mutator_suite_test.go @@ -16,7 +16,7 @@ import ( var suite = builder.NewTestSuiteForMutatingWebhook( mutation.AddToManager, mutation.NewMutator, - "default.mutating.virtualmachineservice.vmoperator.vmware.com") + "default.mutating.virtualmachineservice.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Mutating webhook suite", intgTests, uniTests) diff --git a/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator.go b/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator.go index 1ca888411..d701348c6 100644 --- a/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator.go +++ b/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator.go @@ -48,7 +48,7 @@ var ( ) ) -// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineservice,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachineservices,versions=v1alpha1,name=default.validating.virtualmachineservice.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineservice,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachineservices,versions=v1alpha1,name=default.validating.virtualmachineservice.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservices,verbs=get;list // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservices/status,verbs=get diff --git a/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator_suite_test.go b/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator_suite_test.go index 4abcb9774..23597494a 100644 --- a/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator_suite_test.go +++ b/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator_suite_test.go @@ -16,7 +16,7 @@ import ( var suite = builder.NewTestSuiteForValidatingWebhook( validation.AddToManager, validation.NewValidator, - "default.validating.virtualmachineservice.vmoperator.vmware.com") + "default.validating.virtualmachineservice.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Validation webhook suite", intgTests, unitTests) diff --git a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/mutation/virtualmachinesetresourcepolicy_mutator.go b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/mutation/virtualmachinesetresourcepolicy_mutator.go index 8683b0cf5..354e0f379 100644 --- a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/mutation/virtualmachinesetresourcepolicy_mutator.go +++ b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/mutation/virtualmachinesetresourcepolicy_mutator.go @@ -26,7 +26,7 @@ const ( webHookName = "default" ) -// -kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies,verbs=create;update,versions=v1alpha1,name=default.mutating.virtualmachinesetresourcepolicy.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// -kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies,verbs=create;update,versions=v1alpha1,name=default.mutating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // -kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicy,verbs=get;list // -kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicy/status,verbs=get diff --git a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/mutation/virtualmachinesetresourcepolicy_mutator_suite_test.go b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/mutation/virtualmachinesetresourcepolicy_mutator_suite_test.go index 532c2b0b5..2faf2d8f7 100644 --- a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/mutation/virtualmachinesetresourcepolicy_mutator_suite_test.go +++ b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/mutation/virtualmachinesetresourcepolicy_mutator_suite_test.go @@ -16,7 +16,7 @@ import ( var suite = builder.NewTestSuiteForMutatingWebhook( mutation.AddToManager, mutation.NewMutator, - "default.mutating.virtualmachinesetresourcepolicy.vmoperator.vmware.com") + "default.mutating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Mutating webhook suite", intgTests, uniTests) diff --git a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator.go b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator.go index 3dc96d4b5..8ba2636a3 100644 --- a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator.go +++ b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator.go @@ -31,7 +31,7 @@ const ( webHookName = "default" ) -// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies,versions=v1alpha1,name=default.validating.virtualmachinesetresourcepolicy.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies,versions=v1alpha1,name=default.validating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies,verbs=get;list // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies/status,verbs=get diff --git a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator_suite_test.go b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator_suite_test.go index ae3e39876..3be9d9c7f 100644 --- a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator_suite_test.go +++ b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator_suite_test.go @@ -16,7 +16,7 @@ import ( var suite = builder.NewTestSuiteForValidatingWebhook( validation.AddToManager, validation.NewValidator, - "default.validating.virtualmachinesetresourcepolicy.vmoperator.vmware.com") + "default.validating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Validation webhook suite", intgTests, unitTests) diff --git a/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator.go b/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator.go index 702685904..921e90c04 100644 --- a/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator.go +++ b/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator.go @@ -30,7 +30,7 @@ const ( webHookName = "default" ) -// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-webconsolerequest,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=webconsolerequests,versions=v1alpha1,name=default.validating.webconsolerequest.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-webconsolerequest,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=webconsolerequests,versions=v1alpha1,name=default.validating.webconsolerequest.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=webconsolerequests,verbs=get;list // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=webconsolerequests/status,verbs=get diff --git a/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator_suite_test.go b/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator_suite_test.go index 877e3ce4b..5cbc4c76b 100644 --- a/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator_suite_test.go +++ b/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator_suite_test.go @@ -16,7 +16,7 @@ import ( var suite = builder.NewTestSuiteForValidatingWebhook( validation.AddToManager, validation.NewValidator, - "default.validating.webconsolerequest.vmoperator.vmware.com") + "default.validating.webconsolerequest.v1alpha1.vmoperator.vmware.com") func TestWebhook(t *testing.T) { suite.Register(t, "Validation webhook suite", intgTests, unitTests) From 83f889ed947c740f30e8d32284b49437c5c1f8ed Mon Sep 17 00:00:00 2001 From: akutz Date: Tue, 26 Sep 2023 14:23:36 -0500 Subject: [PATCH 06/54] api: Rm CLS fields from VMI print cols This patch removes the printer columns related to Content Library from the v1a1 VirtualMachineImage and ClusterVirtualMachineImage resources. This is aligned with v1a2, and frankly, these fields provide no real value when printing VMIs and just clutter the output. A user should not care where the image originates, and if they do, they can look at the image's status for more info. --- api/v1alpha1/virtualmachineimage_types.go | 4 ---- .../vmoperator.vmware.com_clustervirtualmachineimages.yaml | 6 ------ .../bases/vmoperator.vmware.com_virtualmachineimages.yaml | 6 ------ 3 files changed, 16 deletions(-) diff --git a/api/v1alpha1/virtualmachineimage_types.go b/api/v1alpha1/virtualmachineimage_types.go index 9f51a7f57..a79c864d5 100644 --- a/api/v1alpha1/virtualmachineimage_types.go +++ b/api/v1alpha1/virtualmachineimage_types.go @@ -156,8 +156,6 @@ func (vmImage *VirtualMachineImage) SetConditions(conditions Conditions) { // +kubebuilder:resource:scope=Cluster,shortName=vmi;vmimage // +kubebuilder:storageversion // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Provider-Name",type="string",JSONPath=".spec.providerRef.name" -// +kubebuilder:printcolumn:name="Content-Library-Name",type="string",JSONPath=".status.contentLibraryRef.name" // +kubebuilder:printcolumn:name="Image-Name",type="string",JSONPath=".status.imageName" // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.productInfo.version" // +kubebuilder:printcolumn:name="Os-Type",type="string",JSONPath=".spec.osInfo.type" @@ -198,8 +196,6 @@ func (clusterVirtualMachineImage *ClusterVirtualMachineImage) SetConditions(cond // +kubebuilder:resource:scope=Cluster,shortName=cvmi;cvmimage;clustervmi;clustervmimage // +kubebuilder:storageversion // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Provider-Name",type="string",JSONPath=".spec.providerRef.name" -// +kubebuilder:printcolumn:name="Content-Library-Name",type="string",JSONPath=".status.contentLibraryRef.name" // +kubebuilder:printcolumn:name="Image-Name",type="string",JSONPath=".status.imageName" // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.productInfo.version" // +kubebuilder:printcolumn:name="Os-Type",type="string",JSONPath=".spec.osInfo.type" diff --git a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml index 9a134a69b..50043573c 100644 --- a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml @@ -21,12 +21,6 @@ spec: scope: Cluster versions: - additionalPrinterColumns: - - jsonPath: .spec.providerRef.name - name: Provider-Name - type: string - - jsonPath: .status.contentLibraryRef.name - name: Content-Library-Name - type: string - jsonPath: .status.imageName name: Image-Name type: string diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml index cfc270ce2..3e314548b 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml @@ -19,12 +19,6 @@ spec: scope: Cluster versions: - additionalPrinterColumns: - - jsonPath: .spec.providerRef.name - name: Provider-Name - type: string - - jsonPath: .status.contentLibraryRef.name - name: Content-Library-Name - type: string - jsonPath: .status.imageName name: Image-Name type: string From 25e4e3890d3c710f1491b7de80500d6944aa07bc Mon Sep 17 00:00:00 2001 From: akutz Date: Wed, 27 Sep 2023 10:59:43 -0500 Subject: [PATCH 07/54] Update mkdocs requirements This patch updates the version of mkdocs, its theme, and deps used to build the documentation. --- docs/requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index a600788ee..3830a4174 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ -mkdocs >= 1.4.2 -mkdocs-material >= 9.1.0 -pymdown-extensions >= 9.9.1 -pygments >= 2.14.0 -mkdocs-git-committers-plugin-2 >= 1.1.1 +mkdocs >= 1.5.3 +mkdocs-material >= 9.4.2 +pymdown-extensions >= 10.3 +pygments >= 2.16.1 +mkdocs-git-committers-plugin-2 >= 1.2.0 mkdocs-git-revision-date-localized-plugin >= 1.2.0 -mkdocs-markdownextradata-plugin >= 0.2.5 \ No newline at end of file +mkdocs-markdownextradata-plugin >= 0.2.5 From ded11080434158071c0a77beba126be438645cb4 Mon Sep 17 00:00:00 2001 From: akutz Date: Wed, 27 Sep 2023 12:27:35 -0500 Subject: [PATCH 08/54] doc: Info about VM image names/resolution This patch adds documentation about VM image names/resolution and creates a dismissable banner to ensure users see that image name logic has changed in vSphere 8.0U2+. --- docs/concepts/images/vm-image.md | 45 ++++++++++++++++++++++++++++++ docs/www/themes/material/main.html | 18 ++++++++++++ mkdocs.yml | 1 + 3 files changed, 64 insertions(+) create mode 100644 docs/www/themes/material/main.html diff --git a/docs/concepts/images/vm-image.md b/docs/concepts/images/vm-image.md index 419d9796a..c080ecf18 100644 --- a/docs/concepts/images/vm-image.md +++ b/docs/concepts/images/vm-image.md @@ -2,6 +2,51 @@ // TODO ([github.com/vmware-tanzu/vm-operator#109](https://github.com/vmware-tanzu/vm-operator/issues/109)) + +## Image Scope + +There are two types of VM image resources, the `ClusterVirtualMachineImage` and `VirtualMachineImage`. The former is a cluster-scoped resource, while the latter is a namespace-scoped resource. Other than that, the two resources are exactly the same. + + +## Image Names + +Prior to vSphere 8.0U2, the name of a VM image resource was derived from the name of a Content Library item. For example, if a Content Library item was named `photonos-5-x64`, then its corresponding `VirtualMachineImage` resource would also be named `photonos-5-x64`. This caused a problem if there library items with the same name from different libraries. With the exception of the first library item encountered, all subsequent library items would have randomly generated data appended to their corresponding Kubernetes resource names to ensure they were unique. In vSphere 8.0U2+, with the introduction of the Image Registry API and potential for global image catalogs, image names needed to be both unique _and_ deterministic, hence: + +``` +vmi-0123456789ABCDEFG +``` + +The above value is referred to as a _VMI ID_, where _VMI_ stands for _Virtual Machine Image_. No matter the source of a VM image, all images have unique, predictable VMI IDs. If the source of a VM image is Content Library, then the VMI ID is constructed using the following steps: + +1. Remove any `-` characters from the Content Library item's UUID +2. Calculate the sha1sum of the value from the previous step +3. Take the first 17 characters from the value from the previous step +4. Append the value from the previous step to `vmi-` + +For example, if the Content Library item's UUID is `e1968c25-dd84-4506-8dc7-9beacb6b688e`, then the VMI ID is `vmi-0a0044d7c690bcbea`, for example: + +1. Remove any `-` characters: `e1968c25dd8445068dc79beacb6b688e`. +1. Get the sha1sum: `0a0044d7c690bcbea07c9b49efc9f743479490e5`. +1. First 17 characters: `0a0044d7c690bcbea`. +1. Create the VMI ID: `vmi-0a0044d7c690bcbea`. + + +## Name Resolution + +When a `VirtualMachine` resource's field `spec.imageName` is set to a VMI ID, the value is resolved to the `VirtualMachineImage` or `ClusterVirtualMachineImage` with that name. It is also possible to specify images based on their _friendly_ name. + +!!! warning "Friendly name resolution" + + Please note that while resolving VM images based on their friendly name was merged into VM Operator with [github.com/vmware-tanzu/vm-operator#214](https://github.com/vmware-tanzu/vm-operator/issues/214), the feature is not yet part of a shipping vSphere release. + +For example, if `vmi-0a0044d7c690bcbea` refers to an image with a friendly name of `photonos-5-x64`, then a user could also specify that value for `spec.imageName` as long as the following is true: + +* There is no other `VirtualMachineImage` in the same namespace with that friendly name. +* There is no other `ClusterVirtualMachineImage` with the same friendly name. + +If the friendly name unambiguously resolves to the distinct, VM image `vmi-0a0044d7c690bcbea`, then a mutation webhook replaces `spec.imageName: photonos-5-x64` with `spec.imageName: vmi-0a0044d7c690bcbea`. + + ## Recommended Images There are no restrictions on the images that can be deployed by VM Operator. However, for users wanting to try things out for themselves, here are a few images the project's developers use on a daily basis: diff --git a/docs/www/themes/material/main.html b/docs/www/themes/material/main.html new file mode 100644 index 000000000..b70cd6960 --- /dev/null +++ b/docs/www/themes/material/main.html @@ -0,0 +1,18 @@ +{% extends "base.html" %} + +{% block announce %} + +
+ + ✨ Please check out + + Image Names + and + + Name Resolution + + to learn about changes to VirtualMachineImage and + ClusterVirtualMachineImage resources in vSphere 8.0U2+ ✨ +
+ +{% endblock %} diff --git a/mkdocs.yml b/mkdocs.yml index e56fa32f0..44de2f58b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -48,6 +48,7 @@ theme: - navigation.path - navigation.indexes - toc.follow + - announce.dismiss extra_css: - www/css/vm-operator.css From 7fbd5362d21be9e7d46fbf8dde254e17c7acc418 Mon Sep 17 00:00:00 2001 From: akutz Date: Thu, 28 Sep 2023 09:33:29 -0500 Subject: [PATCH 09/54] fix: Docs banner resolves relative URL This patch is a small addendum to the previous doc-related change. The relative URL is now correctly resolved regardless of the root of the site. --- docs/www/themes/material/main.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/www/themes/material/main.html b/docs/www/themes/material/main.html index b70cd6960..db7978ad5 100644 --- a/docs/www/themes/material/main.html +++ b/docs/www/themes/material/main.html @@ -6,10 +6,10 @@ ✨ Please check out - Image Names + Image Names and - Name Resolution + Name Resolution to learn about changes to VirtualMachineImage and ClusterVirtualMachineImage resources in vSphere 8.0U2+ ✨ From eb9ac27cee41672fa02013d4ff96688abdcb2acb Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 30 Aug 2023 13:55:55 -0500 Subject: [PATCH 10/54] Add AppendNewExtraConfigValues() --- pkg/util/configspec.go | 23 +++++++++++++++++++++++ pkg/util/configspec_test.go | 22 ++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/pkg/util/configspec.go b/pkg/util/configspec.go index 3d070bad6..97d07d444 100644 --- a/pkg/util/configspec.go +++ b/pkg/util/configspec.go @@ -165,3 +165,26 @@ func RemoveDevicesFromConfigSpec(configSpec *vimTypes.VirtualMachineConfigSpec, } configSpec.DeviceChange = targetDevChanges } + +// AppendNewExtraConfigValues add the new extra config values if not already present in the extra config. +func AppendNewExtraConfigValues( + extraConfig []vimTypes.BaseOptionValue, + newECMap map[string]string) []vimTypes.BaseOptionValue { + + ecMap := make(map[string]vimTypes.AnyType) + for _, opt := range extraConfig { + if optValue := opt.GetOptionValue(); optValue != nil { + ecMap[optValue.Key] = optValue.Value + } + } + + // Only add fields that aren't already in the ExtraConfig. + var newExtraConfig []vimTypes.BaseOptionValue + for k, v := range newECMap { + if _, exists := ecMap[k]; !exists { + newExtraConfig = append(newExtraConfig, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + + return append(extraConfig, newExtraConfig...) +} diff --git a/pkg/util/configspec_test.go b/pkg/util/configspec_test.go index b5cdf2d20..f53594152 100644 --- a/pkg/util/configspec_test.go +++ b/pkg/util/configspec_test.go @@ -214,6 +214,28 @@ var _ = Describe("RemoveDevicesFromConfigSpec", func() { }) }) +var _ = Describe("AppendNewExtraConfigValues", func() { + + It("only adds new values not already in the ExtraConfig", func() { + ec := []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{ + Key: "key1", + Value: "keep-me", + }, + } + + newECMap := map[string]string{ + "key1": "should-be-ignored", + "key2": "add-me", + } + + newExtraConfig := util.AppendNewExtraConfigValues(ec, newECMap) + Expect(newExtraConfig).To(HaveLen(2)) + Expect(newExtraConfig).To(ContainElement(&vimTypes.OptionValue{Key: "key1", Value: "keep-me"})) + Expect(newExtraConfig).To(ContainElement(&vimTypes.OptionValue{Key: "key2", Value: "add-me"})) + }) +}) + var _ = Describe("SanitizeVMClassConfigSpec", func() { oldVMClassAsConfigFSSEnabledFunc := lib.IsVMClassAsConfigFSSEnabled var ( From 517a81aba800ea21c7b4266a58a669216382f922 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 30 Aug 2023 14:02:54 -0500 Subject: [PATCH 11/54] Add GetNetworkProviderType() --- pkg/lib/env.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/lib/env.go b/pkg/lib/env.go index 06591194e..68059b2c0 100644 --- a/pkg/lib/env.go +++ b/pkg/lib/env.go @@ -85,7 +85,11 @@ func GetVMOpNamespaceFromEnv() (string, error) { } var IsNamedNetworkProviderEnabled = func() bool { - return os.Getenv(NetworkProviderType) == NetworkProviderTypeNamed + return GetNetworkProviderType() == NetworkProviderTypeNamed +} + +func GetNetworkProviderType() string { + return os.Getenv(NetworkProviderType) } var IsWcpFaultDomainsFSSEnabled = func() bool { From 57e338042414f8d7bdc1027b5eb83ad2506bd231 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 30 Aug 2023 14:06:53 -0500 Subject: [PATCH 12/54] Initial v1a2 provider There is still a lot of work left to do before switching over to v1a2 but this is able to create TKGs with everything else still only understanding v1a1, so we have a good baseline of version conversion and functionality. Unfortunately the way the existing code is and our need to support the existing v1a1 and new v1a2 APIs in the same codebase makes this a large and painful change. The goal is to wrap up the remaining items as quickly as possible so we can move to v1a2 and then really start to focus on some of our long term tech debt so future version bumps aren't nearly as painful. --- api/v1alpha1/condition_conversion.go | 6 + api/v1alpha1/virtualmachine_conversion.go | 28 +- pkg/conditions2/setter.go | 21 +- pkg/conditions2/setter_test.go | 2 +- pkg/manager/manager.go | 8 +- .../providers/vsphere/vmprovider_vm_test.go | 3 + .../providers/vsphere2/client/client.go | 255 +++ .../vsphere2/client/client_suite_test.go | 46 + .../providers/vsphere2/client/client_test.go | 366 ++++ .../clustermodules/cluster_modules.go | 8 + .../cluster_modules_provider.go | 127 ++ .../cluster_modules_suite_test.go | 26 + .../clustermodules/cluster_modules_test.go | 103 + .../clustermodules/cluster_modules_utils.go | 62 + .../cluster_modules_utils_test.go | 153 ++ .../providers/vsphere2/config/config.go | 290 +++ .../vsphere2/config/config_suite_test.go | 26 + .../providers/vsphere2/config/config_test.go | 211 ++ .../providers/vsphere2/constants/constants.go | 137 ++ .../contentlibrary/content_library.go | 10 + .../content_library_provider.go | 379 ++++ .../content_library_suite_test.go | 26 + .../contentlibrary/content_library_test.go | 139 ++ .../contentlibrary/content_library_utils.go | 149 ++ .../content_library_utils_test.go | 28 + .../vsphere2/credentials/credentials.go | 62 + .../credentials/credentials_suite_test.go | 44 + .../vsphere2/credentials/credentials_test.go | 68 + .../instancestorage/instance_storage.go | 41 + .../providers/vsphere2/internal/internal.go | 59 + .../providers/vsphere2/network/gosc.go | 79 + .../providers/vsphere2/network/gosc_test.go | 132 ++ .../providers/vsphere2/network/netplan.go | 103 + .../vsphere2/network/netplan_test.go | 143 ++ .../providers/vsphere2/network/network.go | 630 ++++++ .../vsphere2/network/network_suite_test.go | 22 + .../vsphere2/network/network_test.go | 344 ++++ .../providers/vsphere2/network/nsxt.go | 117 ++ .../providers/vsphere2/network/nsxt_test.go | 71 + .../vsphere2/placement/cluster_placement.go | 206 ++ .../placement/cluster_placement_test.go | 136 ++ .../placement/placement_suite_test.go | 26 + .../vsphere2/placement/zone_placement.go | 333 +++ .../vsphere2/placement/zone_placement_test.go | 284 +++ .../providers/vsphere2/resources/vm.go | 230 +++ .../providers/vsphere2/session/session.go | 41 + .../vsphere2/session/session_suite_test.go | 22 + .../vsphere2/session/session_util.go | 34 + .../vsphere2/session/session_util_test.go | 93 + .../providers/vsphere2/session/session_vm.go | 60 + .../vsphere2/session/session_vm_update.go | 957 +++++++++ .../session/session_vm_update_test.go | 1236 +++++++++++ .../vsphere2/storage/provisioning.go | 88 + .../vsphere2/storage/storageclass.go | 83 + pkg/vmprovider/providers/vsphere2/test/pki.go | 70 + .../providers/vsphere2/test/suite.go | 61 + .../providers/vsphere2/test/vcsim.go | 31 + .../providers/vsphere2/vcenter/cluster.go | 44 + .../vsphere2/vcenter/cluster_test.go | 47 + .../providers/vsphere2/vcenter/folder.go | 138 ++ .../providers/vsphere2/vcenter/folder_test.go | 173 ++ .../providers/vsphere2/vcenter/getvm.go | 152 ++ .../providers/vsphere2/vcenter/getvm_test.go | 158 ++ .../providers/vsphere2/vcenter/host.go | 41 + .../providers/vsphere2/vcenter/host_test.go | 66 + .../vsphere2/vcenter/resourcepool.go | 163 ++ .../vsphere2/vcenter/resourcepool_test.go | 200 ++ .../vsphere2/vcenter/vcenter_suite_test.go | 30 + .../providers/vsphere2/virtualmachine/ccr.go | 34 + .../vsphere2/virtualmachine/ccr_test.go | 42 + .../vsphere2/virtualmachine/configspec.go | 190 ++ .../virtualmachine/configspec_test.go | 278 +++ .../vsphere2/virtualmachine/conversion.go | 18 + .../virtualmachine/conversion_test.go | 34 + .../vsphere2/virtualmachine/delete.go | 44 + .../vsphere2/virtualmachine/delete_test.go | 71 + .../vsphere2/virtualmachine/devices.go | 100 + .../vsphere2/virtualmachine/heartbeat.go | 25 + .../vsphere2/virtualmachine/publish.go | 64 + .../vsphere2/virtualmachine/publish_test.go | 98 + .../vsphere2/virtualmachine/storage.go | 41 + .../virtualmachine_suite_test.go | 28 + .../virtualmachine/webconsole_ticket.go | 65 + .../virtualmachine/webconsole_ticket_test.go | 43 + .../vsphere2/vmlifecycle/bootstrap.go | 265 +++ .../vmlifecycle/bootstrap_cloudinit.go | 186 ++ .../vmlifecycle/bootstrap_cloudinit_test.go | 398 ++++ .../vmlifecycle/bootstrap_linuxprep.go | 55 + .../vmlifecycle/bootstrap_linuxprep_test.go | 156 ++ .../vsphere2/vmlifecycle/bootstrap_sysprep.go | 79 + .../vmlifecycle/bootstrap_sysprep_test.go | 153 ++ .../vmlifecycle/bootstrap_templatedata.go | 453 ++++ .../bootstrap_templatedata_test.go | 221 ++ .../vsphere2/vmlifecycle/bootstrap_test.go | 51 + .../vmlifecycle/bootstrap_vappconfig.go | 108 + .../vmlifecycle/bootstrap_vappconfig_test.go | 267 +++ .../providers/vsphere2/vmlifecycle/create.go | 41 + .../vsphere2/vmlifecycle/create_clone.go | 188 ++ .../vmlifecycle/create_contentlibrary.go | 105 + .../vsphere2/vmlifecycle/update_status.go | 345 ++++ .../vmlifecycle/update_status_test.go | 211 ++ .../vmlifecycle/vmlifecycle_suite_test.go | 22 + .../providers/vsphere2/vmprovider.go | 428 ++++ .../vsphere2/vmprovider_resourcepolicy.go | 261 +++ .../vmprovider_resourcepolicy_test.go | 234 +++ .../providers/vsphere2/vmprovider_test.go | 88 + .../providers/vsphere2/vmprovider_vm.go | 1169 +++++++++++ .../providers/vsphere2/vmprovider_vm2_test.go | 256 +++ .../providers/vsphere2/vmprovider_vm_test.go | 1813 +++++++++++++++++ .../providers/vsphere2/vmprovider_vm_utils.go | 335 +++ .../vsphere2/vmprovider_vm_utils_test.go | 629 ++++++ .../providers/vsphere2/vsphere_suite_test.go | 31 + test/builder/fake.go | 4 +- test/builder/utila2.go | 275 ++- test/builder/vcsim_test_context.go | 204 +- 115 files changed, 19769 insertions(+), 164 deletions(-) create mode 100644 pkg/vmprovider/providers/vsphere2/client/client.go create mode 100644 pkg/vmprovider/providers/vsphere2/client/client_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/client/client_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go create mode 100644 pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/config/config.go create mode 100644 pkg/vmprovider/providers/vsphere2/config/config_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/config/config_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/constants/constants.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go create mode 100644 pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/credentials/credentials.go create mode 100644 pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go create mode 100644 pkg/vmprovider/providers/vsphere2/internal/internal.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/gosc.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/gosc_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/netplan.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/netplan_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/network.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/network_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/network_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/nsxt.go create mode 100644 pkg/vmprovider/providers/vsphere2/network/nsxt_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/zone_placement.go create mode 100644 pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/resources/vm.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_util.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_util_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_vm.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_vm_update.go create mode 100644 pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/storage/provisioning.go create mode 100644 pkg/vmprovider/providers/vsphere2/storage/storageclass.go create mode 100644 pkg/vmprovider/providers/vsphere2/test/pki.go create mode 100644 pkg/vmprovider/providers/vsphere2/test/suite.go create mode 100644 pkg/vmprovider/providers/vsphere2/test/vcsim.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/cluster.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/folder.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/getvm.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/host.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/host_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go create mode 100644 pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go create mode 100644 pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go diff --git a/api/v1alpha1/condition_conversion.go b/api/v1alpha1/condition_conversion.go index 3211b0c24..275ef4410 100644 --- a/api/v1alpha1/condition_conversion.go +++ b/api/v1alpha1/condition_conversion.go @@ -17,6 +17,12 @@ func Convert_v1alpha1_Condition_To_v1_Condition(in *Condition, out *metav1.Condi out.Reason = in.Reason out.Message = in.Message + // The metav1.Condition requires the reason to be non-empty, when it was not in our prior v1a1 Condition. + // We don't have any great options as to what we can fill this in as. + if out.Reason == "" { + out.Reason = string(out.Status) + } + // TODO: out.ObservedGeneration = return nil diff --git a/api/v1alpha1/virtualmachine_conversion.go b/api/v1alpha1/virtualmachine_conversion.go index 905fb8405..536695bd3 100644 --- a/api/v1alpha1/virtualmachine_conversion.go +++ b/api/v1alpha1/virtualmachine_conversion.go @@ -155,7 +155,7 @@ func convert_v1alpha1_VmMetadata_To_v1alpha2_BootstrapSpec( out.CloudInit = &v1alpha2.VirtualMachineBootstrapCloudInitSpec{ RawCloudConfig: corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: objectName}, - Key: "guestinfo.userdata", + Key: "guestinfo.userdata", // TODO: Is this good enough? v1a1 would include everything with the "guestinfo" prefix }, } case VirtualMachineMetadataOvfEnvTransport: @@ -449,6 +449,12 @@ func convert_v1alpha2_NetworkStatus_To_v1alpha1_Network( func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( in *VirtualMachineSpec, out *v1alpha2.VirtualMachineSpec, s apiconversion.Scope) error { + // The generated auto convert will convert the power modes as-is strings which breaks things, so keep + // this first. + if err := autoConvert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha1_VirtualMachinePowerState_To_v1alpha2_VirtualMachinePowerState(in.PowerState) out.PowerOffMode = convert_v1alpha1_VirtualMachinePowerOpMode_To_v1alpha2_VirtualMachinePowerOpMode(in.PowerOffMode) out.SuspendMode = convert_v1alpha1_VirtualMachinePowerOpMode_To_v1alpha2_VirtualMachinePowerOpMode(in.SuspendMode) @@ -469,12 +475,16 @@ func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( // Deprecated: // in.Ports - return autoConvert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec(in, out, s) + return nil } func Convert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec( in *v1alpha2.VirtualMachineSpec, out *VirtualMachineSpec, s apiconversion.Scope) error { + if err := autoConvert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha2_VirtualMachinePowerState_To_v1alpha1_VirtualMachinePowerState(in.PowerState) out.PowerOffMode = convert_v1alpha2_VirtualMachinePowerOpMode_To_v1alpha1_VirtualMachinePowerOpMode(in.PowerOffMode) out.SuspendMode = convert_v1alpha2_VirtualMachinePowerOpMode_To_v1alpha1_VirtualMachinePowerOpMode(in.SuspendMode) @@ -496,7 +506,7 @@ func Convert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec( // Deprecated: // out.Ports - return autoConvert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec(in, out, s) + return nil } func Convert_v1alpha1_VirtualMachineVolumeStatus_To_v1alpha2_VirtualMachineVolumeStatus( @@ -518,18 +528,26 @@ func Convert_v1alpha2_VirtualMachineVolumeStatus_To_v1alpha1_VirtualMachineVolum func Convert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus( in *VirtualMachineStatus, out *v1alpha2.VirtualMachineStatus, s apiconversion.Scope) error { + if err := autoConvert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha1_VirtualMachinePowerState_To_v1alpha2_VirtualMachinePowerState(in.PowerState) out.Network = convert_v1alpha1_Network_To_v1alpha2_NetworkStatus(in.VmIp, in.NetworkInterfaces) out.LastRestartTime = in.LastRestartTime // WARNING: in.Phase requires manual conversion: does not exist in peer-type - return autoConvert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus(in, out, s) + return nil } func Convert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus( in *v1alpha2.VirtualMachineStatus, out *VirtualMachineStatus, s apiconversion.Scope) error { + if err := autoConvert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus(in, out, s); err != nil { + return err + } + out.PowerState = convert_v1alpha2_VirtualMachinePowerState_To_v1alpha1_VirtualMachinePowerState(in.PowerState) out.Phase = convert_v1alpha2_Conditions_To_v1alpha1_Phase(in.Conditions) out.VmIp, out.NetworkInterfaces = convert_v1alpha2_NetworkStatus_To_v1alpha1_Network(in.Network) @@ -538,7 +556,7 @@ func Convert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus( // WARNING: in.Image requires manual conversion: does not exist in peer-type // WARNING: in.Class requires manual conversion: does not exist in peer-type - return autoConvert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus(in, out, s) + return nil } // ConvertTo converts this VirtualMachine to the Hub version. diff --git a/pkg/conditions2/setter.go b/pkg/conditions2/setter.go index b6f6520dc..0194fe577 100644 --- a/pkg/conditions2/setter.go +++ b/pkg/conditions2/setter.go @@ -78,13 +78,20 @@ func Set(to Setter, condition *metav1.Condition) { func TrueCondition(t string) *metav1.Condition { return &metav1.Condition{ Type: t, - Reason: t, // BMV: This is required field in metav1.Conditions. Fixup API later. Status: metav1.ConditionTrue, + // This is a non-empty field in metav1.Conditions, when it was not in our v1a1 Conditions. This + // really doesn't work with how we've defined our conditions so do something to make things + // work for now. + Reason: string(metav1.ConditionTrue), } } // FalseCondition returns a condition with Status=False and the given type. func FalseCondition(t string, reason string, messageFormat string, messageArgs ...interface{}) *metav1.Condition { + if reason == "" { + reason = string(metav1.ConditionFalse) + } + return &metav1.Condition{ Type: t, Status: metav1.ConditionFalse, @@ -95,6 +102,10 @@ func FalseCondition(t string, reason string, messageFormat string, messageArgs . // UnknownCondition returns a condition with Status=Unknown and the given type. func UnknownCondition(t string, reason string, messageFormat string, messageArgs ...interface{}) *metav1.Condition { + if reason == "" { + reason = string(metav1.ConditionUnknown) + } + return &metav1.Condition{ Type: t, Status: metav1.ConditionUnknown, @@ -124,14 +135,14 @@ func SetSummary(to Setter, options ...MergeOption) { Set(to, summary(to, options...)) } -// SetMirror creates a new condition by mirroring the the Ready condition from a dependent object; -// if the Ready condition does not exists in the source object, no target conditions is generated. +// SetMirror creates a new condition by mirroring the Ready condition from a dependent object; +// if the Ready condition does not exist in the source object, no target conditions is generated. func SetMirror(to Setter, targetCondition string, from Getter, options ...MirrorOptions) { Set(to, mirror(from, targetCondition, options...)) } -// SetAggregate creates a new condition with the aggregation of all the the Ready condition -// from a list of dependent objects; if the Ready condition does not exists in one of the source object, +// SetAggregate creates a new condition with the aggregation of all the Ready condition +// from a list of dependent objects; if the Ready condition does not exist in one of the source object, // the object is excluded from the aggregation; if none of the source object have ready condition, // no target conditions is generated. func SetAggregate(to Setter, targetCondition string, from []Getter, options ...MergeOption) { diff --git a/pkg/conditions2/setter_test.go b/pkg/conditions2/setter_test.go index ceb78d52a..5d9e47b25 100644 --- a/pkg/conditions2/setter_test.go +++ b/pkg/conditions2/setter_test.go @@ -199,7 +199,7 @@ func TestMarkMethods(t *testing.T) { g.Expect(Get(vm, "conditionFoo")).To(haveSameStateOf(&metav1.Condition{ Type: "conditionFoo", Status: metav1.ConditionTrue, - Reason: "conditionFoo", + Reason: "True", })) // test MarkFalse diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index c1d05aca6..374763b4a 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -29,6 +29,7 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/record" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere" + vsphere2 "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" ) // Manager is a VM Operator controller manager. @@ -116,7 +117,12 @@ func New(opts Options) (Manager, error) { func InitializeProviders(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { vmProviderName := fmt.Sprintf("%s/%s/vmProvider", ctx.Namespace, ctx.Name) recorder := record.New(mgr.GetEventRecorderFor(vmProviderName)) - ctx.VMProvider = vsphere.NewVSphereVMProviderFromClient(mgr.GetClient(), recorder) + + if lib.IsVMServiceV1Alpha2FSSEnabled() { + ctx.VMProviderA2 = vsphere2.NewVSphereVMProviderFromClient(mgr.GetClient(), recorder) + } else { + ctx.VMProvider = vsphere.NewVSphereVMProviderFromClient(mgr.GetClient(), recorder) + } return nil } diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go index 899ff5015..0c3d3aa2b 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go @@ -146,6 +146,7 @@ func vmTests() { ) BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed testConfig.WithVMClassAsConfigDaynDate = true ethCard = types.VirtualEthernetCard{ @@ -1590,6 +1591,8 @@ func vmTests() { Context("Multiple NICs are specified", func() { BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + vm.Spec.NetworkInterfaces = []vmopv1.VirtualMachineNetworkInterface{ { NetworkName: "VM Network", diff --git a/pkg/vmprovider/providers/vsphere2/client/client.go b/pkg/vmprovider/providers/vsphere2/client/client.go new file mode 100644 index 000000000..7477fd621 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/client/client.go @@ -0,0 +1,255 @@ +// Copyright (c) 2018-2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client + +import ( + "context" + "net" + "net/url" + "time" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/session/keepalive" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/methods" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +var log = logf.Log.WithName("vsphere").WithName("client") + +type Client struct { + vimClient *vim25.Client + finder *find.Finder + datacenter *object.Datacenter + restClient *rest.Client + contentLibClient contentlibrary.Provider + clusterModClient clustermodules.Provider + sessionManager *session.Manager + config *config.VSphereVMProviderConfig +} + +// Idle time before a keepalive will be invoked. +const keepAliveIdleTime = 5 * time.Minute + +// SoapKeepAliveHandlerFn returns a keepalive handler function suitable for use with the SOAP handler. +// In case the connectivity to VC is down long enough, the session expires. Further attempts to use the +// client yield NotAuthenticated fault. This handler ensures that we re-login the client in those scenarios. +func SoapKeepAliveHandlerFn(sc *soap.Client, sm *session.Manager, userInfo *url.Userinfo) func() error { + return func() error { + ctx := context.Background() + if _, err := methods.GetCurrentTime(ctx, sc); err != nil && isNotAuthenticatedError(err) { + log.Info("Re-authenticating vim client") + if err = sm.Login(ctx, userInfo); err != nil { + if isInvalidLogin(err) { + log.Error(err, "Invalid login in keepalive handler", "url", sc.URL()) + return err + } + } + } else if err != nil { + log.Error(err, "Error in vim25 client's keepalive handler", "url", sc.URL()) + } + + return nil + } +} + +// RestKeepAliveHandlerFn returns a keepalive handler function suitable for use with the REST handler. +// Similar to the SOAP handler, we customize the handler here so we can re-login the client in case the +// REST session expires due to connectivity issues. +func RestKeepAliveHandlerFn(c *rest.Client, userInfo *url.Userinfo) func() error { + return func() error { + ctx := context.Background() + if sess, err := c.Session(ctx); err == nil && sess == nil { + // session is Unauthorized. + log.Info("Re-authenticating REST client") + if err = c.Login(ctx, userInfo); err != nil { + log.Error(err, "Invalid login in keepalive handler", "url", c.URL()) + return err + } + } else if err != nil { + log.Error(err, "Error in rest client's keepalive handler", "url", c.URL()) + } + + return nil + } +} + +// newRestClient creates a rest client which is configured to use a custom keepalive handler function. +func newRestClient(ctx context.Context, vimClient *vim25.Client, config *config.VSphereVMProviderConfig) (*rest.Client, error) { + log.Info("Creating new REST Client", "VcPNID", config.VcPNID, "VcPort", config.VcPort) + restClient := rest.NewClient(vimClient) + + userInfo := url.UserPassword(config.VcCreds.Username, config.VcCreds.Password) + + // Set a custom keepalive handler function + restClient.Transport = keepalive.NewHandlerREST(restClient, keepAliveIdleTime, RestKeepAliveHandlerFn(restClient, userInfo)) + + // Initial login. This will also start the keepalive. + if err := restClient.Login(ctx, userInfo); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, errors.Wrapf(err, "login failed for url: %v", vimClient.URL()) + } + + return restClient, nil +} + +// NewVimClient creates a new vim25 client which is configured to use a custom keepalive handler function. +// Making this public to allow access from other packages when only VimClient is needed. +func NewVimClient(ctx context.Context, config *config.VSphereVMProviderConfig) (*vim25.Client, *session.Manager, error) { + log.Info("Creating new vim Client", "VcPNID", config.VcPNID, "VcPort", config.VcPort) + soapURL, err := soap.ParseURL(net.JoinHostPort(config.VcPNID, config.VcPort)) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse %s:%s", config.VcPNID, config.VcPort) + } + + soapClient := soap.NewClient(soapURL, config.InsecureSkipTLSVerify) + if config.CAFilePath != "" { + err = soapClient.SetRootCAs(config.CAFilePath) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to set root CA %s", config.CAFilePath) + } + } + + vimClient, err := vim25.NewClient(ctx, soapClient) + if err != nil { + return nil, nil, errors.Wrapf(err, "error creating a new vim client for url: %v", soapURL) + } + + if err := vimClient.UseServiceVersion(); err != nil { + return nil, nil, errors.Wrapf(err, "error setting vim client version for url: %v", soapURL) + } + + userInfo := url.UserPassword(config.VcCreds.Username, config.VcCreds.Password) + sm := session.NewManager(vimClient) + + // Set a custom keepalive handler function + vimClient.RoundTripper = keepalive.NewHandlerSOAP(soapClient, keepAliveIdleTime, SoapKeepAliveHandlerFn(soapClient, sm, userInfo)) + + // Initial login. This will also start the keepalive. + if err = sm.Login(ctx, userInfo); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, nil, errors.Wrapf(err, "login failed for url: %v", soapURL) + } + + return vimClient, sm, err +} + +func newFinder( + ctx context.Context, + vimClient *vim25.Client, + config *config.VSphereVMProviderConfig) (*find.Finder, *object.Datacenter, error) { + + finder := find.NewFinder(vimClient, false) + + dcRef, err := finder.ObjectReference(ctx, types.ManagedObjectReference{Type: "Datacenter", Value: config.Datacenter}) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to find Datacenter %q", config.Datacenter) + } + + dc := dcRef.(*object.Datacenter) + finder.SetDatacenter(dc) + + return finder, dc, nil +} + +// NewClient creates a new Client. As a side effect, it creates a vim25 client and a REST client. +func NewClient(ctx context.Context, config *config.VSphereVMProviderConfig) (*Client, error) { + vimClient, sm, err := NewVimClient(ctx, config) + if err != nil { + return nil, err + } + + finder, datacenter, err := newFinder(ctx, vimClient, config) + if err != nil { + return nil, err + } + + restClient, err := newRestClient(ctx, vimClient, config) + if err != nil { + return nil, err + } + + return &Client{ + vimClient: vimClient, + finder: finder, + datacenter: datacenter, + restClient: restClient, + contentLibClient: contentlibrary.NewProvider(restClient), + clusterModClient: clustermodules.NewProvider(restClient), + sessionManager: sm, + config: config, + }, nil +} + +func isNotAuthenticatedError(err error) bool { + if soap.IsSoapFault(err) { + vimFault := soap.ToSoapFault(err).VimFault() + if _, ok := vimFault.(types.NotAuthenticated); ok { + return true + } + } + + return false +} + +func isInvalidLogin(err error) bool { + if soap.IsSoapFault(err) { + vimFault := soap.ToSoapFault(err).VimFault() + if _, ok := vimFault.(types.InvalidLogin); ok { + return true + } + } + + return false +} + +func (c *Client) VimClient() *vim25.Client { + return c.vimClient +} + +func (c *Client) Finder() *find.Finder { + return c.finder +} + +func (c *Client) Datacenter() *object.Datacenter { + return c.datacenter +} + +func (c *Client) RestClient() *rest.Client { + return c.restClient +} + +func (c *Client) ContentLibClient() contentlibrary.Provider { + return c.contentLibClient +} + +func (c *Client) ClusterModuleClient() clustermodules.Provider { + return c.clusterModClient +} + +func (c *Client) Config() *config.VSphereVMProviderConfig { + return c.config +} + +func (c *Client) Logout(ctx context.Context) { + clientURL := c.vimClient.URL() + log.Info("vsphere client logging out from", "VC", clientURL.Host) + if err := c.sessionManager.Logout(ctx); err != nil { + log.Error(err, "Error logging out the vim25 session", "username", clientURL.User.Username(), "host", clientURL.Host) + } + + if err := c.restClient.Logout(ctx); err != nil { + log.Error(err, "Error logging out the rest session", "username", clientURL.User.Username(), "host", clientURL.Host) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/client/client_suite_test.go b/pkg/vmprovider/providers/vsphere2/client/client_suite_test.go new file mode 100644 index 000000000..d77790a4f --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/client/client_suite_test.go @@ -0,0 +1,46 @@ +//go:build !race + +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/test" +) + +var ( + model *simulator.Model + server *simulator.Server + ctx context.Context + tlsTestModel *simulator.Model + tlsServer *simulator.Server + tlsServerCertPath string + tlsServerKeyPath string +) + +var _ = BeforeSuite(func() { + ctx, model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer = test.BeforeSuite() +}) + +var _ = AfterSuite(func() { + test.AfterSuite( + ctx, + model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer) +}) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Client Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/client/client_test.go b/pkg/vmprovider/providers/vsphere2/client/client_test.go new file mode 100644 index 000000000..a5e3aec21 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/client/client_test.go @@ -0,0 +1,366 @@ +//go:build !race + +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package client_test + +import ( + "context" + "net/url" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/session/keepalive" + "github.com/vmware/govmomi/simulator" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" + + . "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/test" +) + +func testConfig(vcpnid string, vcport string, user string, pass string) *config.VSphereVMProviderConfig { + providerConfig := &config.VSphereVMProviderConfig{ + VcPNID: vcpnid, + VcPort: vcport, + Datacenter: "datacenter-2", + VcCreds: &credentials.VSphereVMProviderCredentials{ + Username: user, + Password: pass, + }, + // Let the tests run without TLS validation by default. + InsecureSkipTLSVerify: true, + } + return providerConfig +} + +var _ = Describe("keepalive handler", func() { + + var ( + sessionIdleTimeout = time.Second / 2 + keepAliveIdle = sessionIdleTimeout / 2 + sessionCheckPause = 3 * sessionIdleTimeout // Avoid unit test thread waking up before session expires + simulatorIdleTime time.Duration + ) + + assertSoapSessionValid := func(ctx context.Context, m *session.Manager) { + s, err := m.UserSession(ctx) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, s).NotTo(BeNil()) + } + + assertSoapSessionExpired := func(ctx context.Context, m *session.Manager) { + s, err := m.UserSession(ctx) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, s).To(BeNil()) + } + + BeforeEach(func() { + // Sharing this variable causes the race detector to trip. + simulator.SessionIdleTimeout = sessionIdleTimeout + }) + + AfterEach(func() { + simulator.SessionIdleTimeout = simulatorIdleTime + }) + + Context("for SOAP sessions", func() { + Context("with a logged out session)", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, c *vim25.Client) { + m := session.NewManager(c) + + // Session should be valid since simulator logs in the client by default + assertSoapSessionValid(ctx, m) + + // Sleep for time > sessionIdleTimeout, so the session expires + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertSoapSessionExpired(ctx, m) + + // Set the keepalive handler + c.RoundTripper = keepalive.NewHandlerSOAP(c.RoundTripper, keepAliveIdle, SoapKeepAliveHandlerFn(c.Client, m, simulator.DefaultLogin)) + + // Start the handler + Expect(m.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + time.Sleep(sessionCheckPause) + + // Session should not have been expired + assertSoapSessionValid(ctx, m) + + Expect(m.Logout(ctx)).To(Succeed()) + assertSoapSessionExpired(ctx, m) + }) + }) + }) + + getNewSessionManager := func(url *url.URL) *session.Manager { + c2, err := vim25.NewClient(ctx, soap.NewClient(url, true)) + Expect(err).NotTo(HaveOccurred()) + Expect(c2).NotTo(BeNil()) + + // With default keepalive handler + m2 := session.NewManager(c2) + + c2.RoundTripper = keepalive.NewHandlerSOAP(c2.RoundTripper, keepAliveIdle, nil) + + // Start the handler + Expect(m2.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + return m2 + } + + // To test NotAuthenticated Fault: + // We cannot "self-terminate" a session. So, this following two tests creates _two_ session managers. One with + // the custom keepalive handler, and another session to orchestrate the "NotAuthenticated" fault. + // We terminate the first session using the second session's manager. + + Context("Re-login on NotAuthenticated: with a session that is terminated", func() { + Context("when handler is called with correct userInfo", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, c *vim25.Client) { + + // With custom keepalive handler + m1 := session.NewManager(c) + // Orchestrator session + m2 := getNewSessionManager(c.URL()) + + // Session should be valid + assertSoapSessionValid(ctx, m1) + + // Sleep for time > sessionIdleTimeout + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertSoapSessionExpired(ctx, m1) + + // set the keepalive handler + c.RoundTripper = keepalive.NewHandlerSOAP(c.RoundTripper, keepAliveIdle, SoapKeepAliveHandlerFn(c.Client, m1, simulator.DefaultLogin)) + + // Start the handler + Expect(m1.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Wait for the keepalive handler to get called + time.Sleep(sessionCheckPause) + + // Session should be valid since Login starts a new one + assertSoapSessionValid(ctx, m1) + + // Terminate session to emulate NotAuthenticated Error + sess, err := m1.UserSession(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(sess).NotTo(BeNil()) + + By("terminating the session") + Expect(m2.TerminateSession(ctx, []string{sess.Key})).To(Succeed()) + + // session expired since it is terminated + assertSoapSessionExpired(ctx, m1) + + time.Sleep(sessionCheckPause) + + // keepalive handler must have re-logged in + assertSoapSessionValid(ctx, m1) + }) + }) + }) + }) + + Context("Relogin on NotAuthenticated: with a session that is terminated", func() { + Context("when handler is called with wrong userInfo", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, c *vim25.Client) { + + // With custom keepalive handler + m1 := session.NewManager(c) + // With default keepalive handler + m2 := getNewSessionManager(c.URL()) + + // Session should be valid + assertSoapSessionValid(ctx, m1) + + // Sleep for time > sessionIdleTimeout + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertSoapSessionExpired(ctx, m1) + + // set the keepalive handler with wrong userInfo + c.RoundTripper = keepalive.NewHandlerSOAP(c.RoundTripper, keepAliveIdle, SoapKeepAliveHandlerFn(c.Client, m1, nil)) + + // Start the handler + Expect(m1.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Wait for the keepalive handler to get called + time.Sleep(sessionCheckPause) + + // Session should be valid since Login starts a new one + assertSoapSessionValid(ctx, m1) + + // Terminate session to emulate NotAuthenticated Error + sess, err := m1.UserSession(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(sess).NotTo(BeNil()) + + By("terminating the session") + Expect(m2.TerminateSession(ctx, []string{sess.Key})).To(Succeed()) + + // session expired since it is terminated + assertSoapSessionExpired(ctx, m1) + + // Wait for keepalive handler to be called + time.Sleep(sessionCheckPause) + + // keepalive handler should error out, session still invalid + assertSoapSessionExpired(ctx, m1) + }) + }) + }) + }) + }) + + assertRestSessionValid := func(ctx context.Context, c *rest.Client) { + s, err := c.Session(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(s).NotTo(BeNil()) + } + + assertRestSessionExpired := func(ctx context.Context, c *rest.Client) { + s, err := c.Session(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(s).To(BeNil()) + } + + Context("REST sessions", func() { + Context("When a REST session is logged out)", func() { + It("re-logins the session", func() { + simulator.Test(func(ctx context.Context, vc *vim25.Client) { + c := rest.NewClient(vc) + + Expect(c.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Session should be valid + assertRestSessionValid(ctx, c) + + // Sleep for time > sessionIdleTimeout + time.Sleep(sessionCheckPause) + + // Session should be expired now + assertRestSessionExpired(ctx, c) + + // Set the keepalive handler + c.Transport = keepalive.NewHandlerREST(c, keepAliveIdle, RestKeepAliveHandlerFn(c, simulator.DefaultLogin)) + + // Start the handler + Expect(c.Login(ctx, simulator.DefaultLogin)).To(Succeed()) + + // Session should not have been expired + assertRestSessionValid(ctx, c) + + Expect(c.Logout(ctx)).To(Succeed()) + assertRestSessionExpired(ctx, c) + }) + }) + }) + }) +}) + +var _ = Describe("NewClient", func() { + Context("When called with valid config", func() { + Specify("returns a valid client and no error", func() { + client, err := NewClient(ctx, testConfig(server.URL.Hostname(), server.URL.Port(), "some-username", "some-password")) + Expect(err).ToNot(HaveOccurred()) + Expect(client).ToNot(BeNil()) + }) + }) + + Context("When called with invalid host and port", func() { + Specify("soap.ParseURL should fail", func() { + failConfig := testConfig("test%test", "", "test-user", "test-pass") + client, err := NewClient(ctx, failConfig) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("failed to parse")) + Expect(client).To(BeNil()) + }) + }) + + Context("When called with invalid VC PNID", func() { + Specify("returns failed to parse error", func() { + failConfig := testConfig("test-pnid", "test-port", "test-user", "test-pass") + client, err := NewClient(ctx, failConfig) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid")) + Expect(err.Error()).To(ContainSubstring("port")) + Expect(err.Error()).To(ContainSubstring("test-port")) + Expect(client).To(BeNil()) + }) + }) + + DescribeTable("Should fail if given wrong username and/or wrong password", + func(expectedUsername, expectedPassword, username, password string) { + server.URL.User = url.UserPassword(expectedUsername, expectedPassword) + model.Service.Listen = server.URL + config := testConfig(server.URL.Hostname(), server.URL.Port(), username, password) + client, err := NewClient(ctx, config) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("login failed for url")) + Expect(client).To(BeNil()) + }, + Entry("with wrong username and password", "correct-username", "correct-password", "username", "password"), + Entry("with wrong username and correct password", "correct-username", "correct-password", "username", "correct-password"), + Entry("with correct username and wrong password", "correct-username", "correct-password", "correct-username", "password"), + ) +}) + +// Most of the other VM Operator tests run without TLS verification. Start up a separate simulator with a fresh TLS key/cert +// +// and ensure the client can connect to it. +var _ = Describe("Tests for client TLS", func() { + Context("when the client recognizes the certificate presented by the VC", func() { + It("successfully connects to the VC", func() { + tlsServer.URL.User = url.UserPassword("some-username", "some-password") + config := testConfig(tlsServer.URL.Hostname(), tlsServer.URL.Port(), "some-username", "some-password") + config.InsecureSkipTLSVerify = false + config.CAFilePath = tlsServerCertPath + _, err := NewClient(context.Background(), config) + + Expect(err).NotTo(HaveOccurred()) + }) + }) + Context("when the CA bundle referred to does not exist", func() { + It("returns an error about loading the CA bundle", func() { + tlsServer.URL.User = url.UserPassword("some-username", "some-password") + config := testConfig(tlsServer.URL.Hostname(), tlsServer.URL.Port(), "some-username", "some-password") + config.CAFilePath = "/a/nonexistent/ca-bundle.crt" + config.InsecureSkipTLSVerify = false + _, err := NewClient(context.Background(), config) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to set root CA")) + }) + + }) + Context("when the client does not recognize the certificate presented by the VC", func() { + Context("when TLS verification is used", func() { + It("returns an error about certificate validation", func() { + tlsServer.URL.User = url.UserPassword("some-username", "some-password") + config := testConfig(tlsServer.URL.Hostname(), tlsServer.URL.Port(), "some-username", "some-password") + _, randomCANotForServer := test.GenerateSelfSignedCert() + config.CAFilePath = randomCANotForServer + config.InsecureSkipTLSVerify = false + _, err := NewClient(context.Background(), config) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("x509: certificate signed by unknown authority")) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go new file mode 100644 index 000000000..7e8208443 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules.go @@ -0,0 +1,8 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules + +import logf "sigs.k8s.io/controller-runtime/pkg/log" + +var log = logf.Log.WithName("vsphere").WithName("clustermodules") diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go new file mode 100644 index 000000000..ce21d3dd6 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_provider.go @@ -0,0 +1,127 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules + +import ( + "context" + + "github.com/vmware/govmomi/vapi/cluster" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +type Provider interface { + CreateModule(ctx context.Context, clusterRef types.ManagedObjectReference) (string, error) + DeleteModule(ctx context.Context, moduleID string) error + DoesModuleExist(ctx context.Context, moduleID string, cluster types.ManagedObjectReference) (bool, error) + + IsMoRefModuleMember(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) (bool, error) + AddMoRefToModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error + RemoveMoRefFromModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error +} + +type provider struct { + manager *cluster.Manager +} + +func NewProvider(restClient *rest.Client) Provider { + return &provider{ + manager: cluster.NewManager(restClient), + } +} + +func (cm *provider) CreateModule(ctx context.Context, clusterRef types.ManagedObjectReference) (string, error) { + log.Info("Creating cluster module", "cluster", clusterRef) + + moduleID, err := cm.manager.CreateModule(ctx, clusterRef) + if err != nil { + return "", err + } + + log.Info("Created cluster module", "moduleID", moduleID) + return moduleID, nil +} + +func (cm *provider) DeleteModule(ctx context.Context, moduleID string) error { + log.Info("Deleting cluster module", "moduleID", moduleID) + + err := cm.manager.DeleteModule(ctx, moduleID) + if err != nil && !lib.IsNotFoundError(err) { + return err + } + + log.Info("Deleted cluster module", "moduleID", moduleID) + return nil +} + +func (cm *provider) DoesModuleExist(ctx context.Context, moduleID string, clusterRef types.ManagedObjectReference) (bool, error) { + log.V(4).Info("Checking if cluster module exists", "moduleID", moduleID, "clusterRef", clusterRef) + + if moduleID == "" { + return false, nil + } + + // This is not efficient for as we use DoesModuleExist(). + modules, err := cm.manager.ListModules(ctx) + if err != nil { + return false, err + } + + for _, mod := range modules { + if mod.Cluster == clusterRef.Value && mod.Module == moduleID { + return true, nil + } + } + + log.V(4).Info("Cluster module doesn't exist", "moduleID", moduleID, "clusterRef", clusterRef) + return false, nil +} + +func (cm *provider) IsMoRefModuleMember(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) (bool, error) { + moduleMembers, err := cm.manager.ListModuleMembers(ctx, moduleID) + if err != nil { + return false, err + } + + for _, member := range moduleMembers { + if member.Reference() == moRef.Reference() { + return true, nil + } + } + + return false, nil +} + +func (cm *provider) AddMoRefToModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error { + isMember, err := cm.IsMoRefModuleMember(ctx, moduleID, moRef) + if err != nil { + return err + } + + if !isMember { + log.Info("Adding moRef to cluster module", "moduleID", moduleID, "moRef", moRef) + // TODO: Should we just skip the IsMoRefModuleMember() and always call this since we're already + // ignoring the first return value? + _, err := cm.manager.AddModuleMembers(ctx, moduleID, moRef.Reference()) + if err != nil { + return err + } + } + + return nil +} + +func (cm *provider) RemoveMoRefFromModule(ctx context.Context, moduleID string, moRef types.ManagedObjectReference) error { + log.Info("Removing moRef from cluster module", "moduleID", moduleID, "moRef", moRef) + + _, err := cm.manager.RemoveModuleMembers(ctx, moduleID, moRef) + if err != nil { + return err + } + + log.Info("Removed moRef from cluster module", "moduleID", moduleID, "moRef", moRef) + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go new file mode 100644 index 000000000..e46a72346 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("ClusterModules Provider", cmTests) +} + +var suite = builder.NewTestSuite() + +func TestClusterModules(t *testing.T) { + suite.Register(t, "vSphere Provider Cluster Modules Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go new file mode 100644 index 000000000..67ce19011 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func cmTests() { + Describe("Cluster Modules", func() { + + var ( + ctx *builder.TestContextForVCSim + cmProvider clustermodules.Provider + + moduleGroup string + moduleSpec *vmopv1.ClusterModuleSpec + moduleStatus *vmopv1.ClusterModuleStatus + clusterRef types.ManagedObjectReference + vmRef types.ManagedObjectReference + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + cmProvider = clustermodules.NewProvider(ctx.RestClient) + + clusterRef = ctx.GetSingleClusterCompute().Reference() + + moduleGroup = "controller-group" + moduleSpec = &vmopv1.ClusterModuleSpec{ + GroupName: moduleGroup, + } + + moduleID, err := cmProvider.CreateModule(ctx, clusterRef) + Expect(err).NotTo(HaveOccurred()) + Expect(moduleID).ToNot(BeEmpty()) + + moduleStatus = &vmopv1.ClusterModuleStatus{ + GroupName: moduleSpec.GroupName, + ModuleUuid: moduleID, + } + + // TODO: Create VM instead of using one that vcsim creates for free. + vm, err := ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + vmRef = vm.Reference() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Create a ClusterModule, verify it exists and delete it", func() { + exists, err := cmProvider.DoesModuleExist(ctx, moduleStatus.ModuleUuid, clusterRef) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + + Expect(cmProvider.DeleteModule(ctx, moduleStatus.ModuleUuid)).To(Succeed()) + + exists, err = cmProvider.DoesModuleExist(ctx, moduleStatus.ModuleUuid, clusterRef) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + + Context("ClusterModule-VM association", func() { + It("check membership doesn't exist", func() { + isMember, err := cmProvider.IsMoRefModuleMember(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + Expect(isMember).To(BeFalse()) + }) + + It("Associate a VM with a clusterModule, check the membership and remove it", func() { + By("Associate VM") + err := cmProvider.AddMoRefToModule(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + + By("Verify membership") + isMember, err := cmProvider.IsMoRefModuleMember(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + Expect(isMember).To(BeTrue()) + + By("Remove the association") + err = cmProvider.RemoveMoRefFromModule(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + + By("Verify no longer a member") + isMember, err = cmProvider.IsMoRefModuleMember(ctx, moduleStatus.ModuleUuid, vmRef) + Expect(err).NotTo(HaveOccurred()) + Expect(isMember).To(BeFalse()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go new file mode 100644 index 000000000..898a7d82e --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils.go @@ -0,0 +1,62 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules + +import ( + "context" + + "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +// FindClusterModuleUUID returns the index in the Status.ClusterModules and UUID of the +// VC cluster module for the given groupName and cluster reference. +func FindClusterModuleUUID( + groupName string, + clusterRef types.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (int, string) { + + // Prior to the stretched cluster work, the status did not contain the VC cluster the module was + // created for, but we still need to return existing modules when the FSS is not enabled. + matchCluster := lib.IsWcpFaultDomainsFSSEnabled() + + for i, modStatus := range resourcePolicy.Status.ClusterModules { + if modStatus.GroupName == groupName && (modStatus.ClusterMoID == clusterRef.Value || !matchCluster) { + return i, modStatus.ModuleUuid + } + } + + return -1, "" +} + +// ClaimClusterModuleUUID tries to find an existing entry in the Status.ClusterModules that is for +// the given groupName and cluster reference. This is meant for after an upgrade where the FaultDomains +// FSS is now enabled but we had not previously set the ClusterMoID. +func ClaimClusterModuleUUID( + ctx context.Context, + clusterModProvider Provider, + groupName string, + clusterRef types.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (int, string, error) { + + var errs []error + + if lib.IsWcpFaultDomainsFSSEnabled() { + for i, modStatus := range resourcePolicy.Status.ClusterModules { + if modStatus.GroupName == groupName && modStatus.ClusterMoID == "" { + exists, err := clusterModProvider.DoesModuleExist(ctx, modStatus.ModuleUuid, clusterRef) + if err != nil { + errs = append(errs, err) + } else if exists { + return i, modStatus.ModuleUuid, nil + } + } + } + } + + return -1, "", k8serrors.NewAggregate(errs) +} diff --git a/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go new file mode 100644 index 000000000..db3e83241 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/clustermodules/cluster_modules_utils_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustermodules_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" +) + +var _ = Describe("FindClusterModuleUUID", func() { + const ( + groupName1, groupName2 = "groupName1", "groupName2" + moduleUUID1, moduleUUID2 = "uuid1", "uuid2" + ) + + var ( + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + clusterRef1, clusterRef2 types.ManagedObjectReference + ) + + BeforeEach(func() { + resourcePolicy = &vmopv1.VirtualMachineSetResourcePolicy{ + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ClusterModuleGroups: []string{groupName1, groupName2}, + }, + } + + clusterRef1 = types.ManagedObjectReference{Value: "dummy-cluster1"} + clusterRef2 = types.ManagedObjectReference{Value: "dummy-cluster2"} + }) + + Context("FaultDomains FSS is disabled", func() { + + Context("GroupName does not exist", func() { + It("Returns expected values", func() { + idx, uuid := clustermodules.FindClusterModuleUUID("does-not-exist", clusterRef1, resourcePolicy) + Expect(idx).To(Equal(-1)) + Expect(uuid).To(BeEmpty()) + }) + }) + + Context("GroupName exists", func() { + BeforeEach(func() { + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID1, + }, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName2, + ModuleUuid: moduleUUID2, + ClusterMoID: "this should be ignored", + }, + ) + }) + + It("Returns expected entry", func() { + idx, uuid := clustermodules.FindClusterModuleUUID(groupName1, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(0)) + Expect(uuid).To(Equal(moduleUUID1)) + + idx, uuid = clustermodules.FindClusterModuleUUID(groupName2, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(1)) + Expect(uuid).To(Equal(moduleUUID2)) + }) + }) + + }) + + Context("FaultDomains FSS is enabled", func() { + var ( + oldFaultDomainsFunc func() bool + ) + + BeforeEach(func() { + oldFaultDomainsFunc = lib.IsWcpFaultDomainsFSSEnabled + lib.IsWcpFaultDomainsFSSEnabled = func() bool { return true } + }) + + AfterEach(func() { + lib.IsWcpFaultDomainsFSSEnabled = oldFaultDomainsFunc + }) + + Context("GroupName does not exist", func() { + It("Returns expected values", func() { + idx, uuid := clustermodules.FindClusterModuleUUID("does-not-exist", clusterRef1, resourcePolicy) + Expect(idx).To(Equal(-1)) + Expect(uuid).To(BeEmpty()) + }) + }) + + Context("GroupName exists", func() { + BeforeEach(func() { + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID1, + ClusterMoID: clusterRef1.Value, + }, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName2, + ModuleUuid: moduleUUID2, + ClusterMoID: clusterRef1.Value, + }, + ) + }) + + It("Returns expected entry", func() { + idx, uuid := clustermodules.FindClusterModuleUUID(groupName1, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(0)) + Expect(uuid).To(Equal(moduleUUID1)) + + idx, uuid = clustermodules.FindClusterModuleUUID(groupName2, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(1)) + Expect(uuid).To(Equal(moduleUUID2)) + }) + }) + + Context("Matches by cluster reference", func() { + BeforeEach(func() { + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID1, + ClusterMoID: clusterRef1.Value, + }, + vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName1, + ModuleUuid: moduleUUID2, + ClusterMoID: clusterRef2.Value, + }, + ) + }) + + It("Returns expected entry", func() { + idx, uuid := clustermodules.FindClusterModuleUUID(groupName1, clusterRef1, resourcePolicy) + Expect(idx).To(Equal(0)) + Expect(uuid).To(Equal(moduleUUID1)) + + idx, uuid = clustermodules.FindClusterModuleUUID(groupName1, clusterRef2, resourcePolicy) + Expect(idx).To(Equal(1)) + Expect(uuid).To(Equal(moduleUUID2)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/config/config.go b/pkg/vmprovider/providers/vsphere2/config/config.go new file mode 100644 index 000000000..8d2071a42 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/config/config.go @@ -0,0 +1,290 @@ +// Copyright (c) 2018-2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "context" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/pkg/errors" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" +) + +var log = logf.Log.WithName("vsphere").WithName("config") + +// VSphereVMProviderConfig represents the configuration for a Vsphere VM Provider instance. +// Contains information enabling integration with a backend vSphere instance for VM management. +type VSphereVMProviderConfig struct { + VcPNID string + VcPort string + VcCreds *credentials.VSphereVMProviderCredentials + Datacenter string + StorageClassRequired bool // Always true in WCP env. + UseInventoryAsContentSource bool // Always false in WCP env. + CAFilePath string + InsecureSkipTLSVerify bool // Always false in WCP env. + + // These are Zone and/or Namespace specific. + ResourcePool string + Folder string + + // Only set in simulated testing env. + Datastore string + Network string +} + +const ( + DefaultVCPort = "443" + + ProviderConfigMapName = "vsphere.provider.config.vmoperator.vmware.com" + // Keys in provider ConfigMap. + vcPNIDKey = "VcPNID" + vcPortKey = "VcPort" + vcCredsSecretNameKey = "VcCredsSecretName" //nolint:gosec + datacenterKey = "Datacenter" + resourcePoolKey = "ResourcePool" + folderKey = "Folder" + datastoreKey = "Datastore" + networkNameKey = "Network" + scRequiredKey = "StorageClassRequired" + useInventoryKey = "UseInventoryAsContentSource" + insecureSkipTLSVerifyKey = "InsecureSkipTLSVerify" + caFilePathKey = "CAFilePath" + ContentSourceKey = "ContentSource" + + NetworkConfigMapName = "vmoperator-network-config" + NameserversKey = "nameservers" // Key in the NetworkConfigMapName. + SearchSuffixesKey = "searchsuffixes" // Key in the NetworkConfigMapName. +) + +// ConfigMapToProviderConfig converts the VM provider ConfigMap to a VSphereVMProviderConfig. +func ConfigMapToProviderConfig( //nolint: revive // Ignore linter error about stuttering. + configMap *corev1.ConfigMap, + vcCreds *credentials.VSphereVMProviderCredentials) (*VSphereVMProviderConfig, error) { + + vcPNID, ok := configMap.Data[vcPNIDKey] + if !ok { + return nil, errors.New("missing configMap data field VcPNID") + } + + vcPort, ok := configMap.Data[vcPortKey] + if !ok { + vcPort = DefaultVCPort + } + + scRequired := false + if s, ok := configMap.Data[scRequiredKey]; ok { + var err error + scRequired, err = strconv.ParseBool(s) + if err != nil { + return nil, errors.Wrap(err, "unable to parse value of StorageClassRequired") + } + } + + useInventory := false + if u, ok := configMap.Data[useInventoryKey]; ok { + var err error + useInventory, err = strconv.ParseBool(u) + if err != nil { + return nil, errors.Wrap(err, "unable to parse value of UseInventory") + } + } + + // Default to validating TLS. + insecureSkipTLSVerify := false + if v, ok := configMap.Data[insecureSkipTLSVerifyKey]; ok { + var err error + insecureSkipTLSVerify, err = strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrap(err, "unable to parse value of InsecureSkipTLSVerify") + } + } + + var caFilePath string + if ca, ok := configMap.Data[caFilePathKey]; !insecureSkipTLSVerify && ok { + // The value will be /etc/vmware/wcp/tls/vmca.pem. While this is from our provider ConfigMap + // it must match the volume path in our Deployment. + caFilePath = ca + } + + ret := &VSphereVMProviderConfig{ + VcPNID: vcPNID, + VcPort: vcPort, + VcCreds: vcCreds, + Datacenter: configMap.Data[datacenterKey], + ResourcePool: configMap.Data[resourcePoolKey], + Folder: configMap.Data[folderKey], + Datastore: configMap.Data[datastoreKey], + Network: configMap.Data[networkNameKey], + StorageClassRequired: scRequired, + UseInventoryAsContentSource: useInventory, + InsecureSkipTLSVerify: insecureSkipTLSVerify, + CAFilePath: caFilePath, + } + + return ret, nil +} + +func configMapToProviderCredentials( + client ctrlruntime.Client, + configMap *corev1.ConfigMap) (*credentials.VSphereVMProviderCredentials, error) { + + secretName := configMap.Data[vcCredsSecretNameKey] + if secretName == "" { + return nil, errors.Errorf("%s creds secret not set in vmop system namespace", vcCredsSecretNameKey) + } + + return credentials.GetProviderCredentials(client, configMap.Namespace, secretName) +} + +func GetDNSInformationFromConfigMap(client ctrlruntime.Client) ([]string, []string, error) { + vmopNamespace, err := lib.GetVMOpNamespaceFromEnv() + if err != nil { + return nil, nil, err + } + + configMap := &corev1.ConfigMap{} + configMapKey := ctrlruntime.ObjectKey{Name: NetworkConfigMapName, Namespace: vmopNamespace} + if err := client.Get(context.Background(), configMapKey, configMap); err != nil { + return nil, nil, err + } + + var ( + nameservers []string + searchSuffixes []string + ) + + nsStr, ok := configMap.Data[NameserversKey] + if !ok { + return nil, nil, errors.Wrapf(err, "invalid %v ConfigMap, missing key nameservers", NetworkConfigMapName) + } + + nameservers = strings.Fields(nsStr) + if len(nameservers) == 0 { + return nil, nil, errors.Errorf("No nameservers in %v ConfigMap", NetworkConfigMapName) + } + + if len(nameservers) == 1 && nameservers[0] == "" { + return nil, nil, errors.Errorf("No valid nameservers in %v ConfigMap. It still contains key", NetworkConfigMapName) + } + + if ssStr, ok := configMap.Data[SearchSuffixesKey]; ok { + searchSuffixes = strings.Fields(ssStr) + } + + // do we need to validate that these look like valid ipv4 addresses? + return nameservers, searchSuffixes, nil +} + +// getProviderConfigMap returns the provider ConfigMap. +func getProviderConfigMap( + ctx context.Context, + client ctrlruntime.Client) (*corev1.ConfigMap, error) { + + vmopNamespace, err := lib.GetVMOpNamespaceFromEnv() + if err != nil { + return nil, err + } + + configMap := &corev1.ConfigMap{} + configMapKey := ctrlruntime.ObjectKey{Name: ProviderConfigMapName, Namespace: vmopNamespace} + if err := client.Get(ctx, configMapKey, configMap); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, errors.Wrapf(err, "error retrieving the provider ConfigMap %s", configMapKey) + } + + return configMap, nil +} + +// GetProviderConfig returns a provider config constructed from vSphere Provider ConfigMap in the VM Operator namespace. +func GetProviderConfig( + ctx context.Context, + client ctrlruntime.Client) (*VSphereVMProviderConfig, error) { + + configMap, err := getProviderConfigMap(ctx, client) + if err != nil { + return nil, err + } + + vcCreds, err := configMapToProviderCredentials(client, configMap) + if err != nil { + return nil, err + } + + providerConfig, err := ConfigMapToProviderConfig(configMap, vcCreds) + if err != nil { + return nil, err + } + + return providerConfig, nil +} + +func setConfigMapData(configMap *corev1.ConfigMap, config *VSphereVMProviderConfig, vcCredsSecretName string) { + if configMap.Data == nil { + configMap.Data = map[string]string{} + } + + configMap.Data[vcPNIDKey] = config.VcPNID + configMap.Data[vcPortKey] = config.VcPort + configMap.Data[vcCredsSecretNameKey] = vcCredsSecretName + configMap.Data[datacenterKey] = config.Datacenter + configMap.Data[resourcePoolKey] = config.ResourcePool + configMap.Data[folderKey] = config.Folder + configMap.Data[datastoreKey] = config.Datastore + configMap.Data[scRequiredKey] = strconv.FormatBool(config.StorageClassRequired) + configMap.Data[useInventoryKey] = strconv.FormatBool(config.UseInventoryAsContentSource) + configMap.Data[caFilePathKey] = config.CAFilePath + configMap.Data[insecureSkipTLSVerifyKey] = strconv.FormatBool(config.InsecureSkipTLSVerify) +} + +// ProviderConfigToConfigMap returns the ConfigMap for the config. +// Used only in testing. +func ProviderConfigToConfigMap( + namespace string, + config *VSphereVMProviderConfig, + vcCredsSecretName string) *corev1.ConfigMap { + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ProviderConfigMapName, + Namespace: namespace, + }, + } + setConfigMapData(configMap, config, vcCredsSecretName) + + return configMap +} + +// UpdateVcInConfigMap updates the ConfigMap with the new vCenter PNID and Port. Returns false if no updated needed. +func UpdateVcInConfigMap(ctx context.Context, client ctrlruntime.Client, vcPNID, vcPort string) (bool, error) { + configMap, err := getProviderConfigMap(ctx, client) + if err != nil { + return false, err + } + + if configMap.Data[vcPNIDKey] == vcPNID && configMap.Data[vcPortKey] == vcPort { + // No update needed. + return false, nil + } + + origConfigMap := configMap.DeepCopy() + configMap.Data[vcPNIDKey] = vcPNID + configMap.Data[vcPortKey] = vcPort + + err = client.Patch(ctx, configMap, ctrlruntime.MergeFrom(origConfigMap)) + if err != nil { + log.Error(err, "Failed to update provider ConfigMap", "configMapName", configMap.Name) + return false, err + } + + return true, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/config/config_suite_test.go b/pkg/vmprovider/providers/vsphere2/config/config_suite_test.go new file mode 100644 index 000000000..c82c0e933 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/config/config_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("Config", configTests) +} + +var suite = builder.NewTestSuite() + +func TestConfig(t *testing.T) { + suite.Register(t, "vSphere Provider Config Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/config/config_test.go b/pkg/vmprovider/providers/vsphere2/config/config_test.go new file mode 100644 index 000000000..28bda78bb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/config/config_test.go @@ -0,0 +1,211 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package config_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func configTests() { + + var ( + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Describe("GetProviderConfig", func() { + + Context("GetProviderConfig", func() { + + Context("when a secret doesn't exist", func() { + It("returns no provider config and an error", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vmop-vcsim-dummy-creds", + Namespace: ctx.PodNamespace, + }, + } + Expect(ctx.Client.Delete(ctx, secret)).To(Succeed()) + + providerConfig, err := config.GetProviderConfig(ctx, ctx.Client) + Expect(err).To(HaveOccurred()) + Expect(providerConfig).To(BeNil()) + }) + }) + + Context("when a secret exists", func() { + It("returns a good provider config", func() { + _, err := config.GetProviderConfig(ctx, ctx.Client) + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) + }) + + Describe("UpdateVcInConfigMap", func() { + + Context("UpdateVcInConfigMap", func() { + DescribeTable("Update VC PNID and VC Port", + func(newPnid string, newPort string) { + expectedUpdated := newPnid != "" || newPort != "" + + providerConfig, err := config.GetProviderConfig(ctx, ctx.Client) + Expect(err).ToNot(HaveOccurred()) + if newPnid == "" { + newPnid = providerConfig.VcPNID + } + if newPort == "" { + newPort = providerConfig.VcPort + } + + updated, err := config.UpdateVcInConfigMap(ctx, ctx.Client, newPnid, newPort) + Expect(err).ToNot(HaveOccurred()) + Expect(updated).To(Equal(expectedUpdated)) + + providerConfig, err = config.GetProviderConfig(ctx, ctx.Client) + Expect(err).NotTo(HaveOccurred()) + Expect(providerConfig.VcPNID).To(Equal(newPnid)) + Expect(providerConfig.VcPort).To(Equal(newPort)) + }, + Entry("only VC PNID is updated", "some-pnid", nil), + Entry("only VC Port is updated", nil, "some-port"), + Entry("both VC PNID and Port are updated", "some-pnid", "some-port"), + Entry("neither VC PNID and Port are updated", nil, nil), + ) + }) + }) +} + +var _ = Describe("ConfigMapToProviderConfig", func() { + + var ( + providerCreds *credentials.VSphereVMProviderCredentials + providerConfigIn *config.VSphereVMProviderConfig + configMap *corev1.ConfigMap + ) + + BeforeEach(func() { + providerCreds = &credentials.VSphereVMProviderCredentials{Username: "username", Password: "password"} + providerConfigIn = &config.VSphereVMProviderConfig{ + VcPNID: "my-vc.vmware.com", + VcPort: "433", + VcCreds: providerCreds, + Datacenter: "datacenter-42", + StorageClassRequired: false, + UseInventoryAsContentSource: false, + CAFilePath: "/etc/pki/tls/certs/ca-bundle.crt", + InsecureSkipTLSVerify: false, + ResourcePool: "resourcepool-42", + Folder: "folder-42", + Datastore: "/DC0/datastore/LocalDS_0", + } + }) + + JustBeforeEach(func() { + configMap = config.ProviderConfigToConfigMap("dummy-ns", providerConfigIn, "dummy-secrets") + }) + + It("provider config is correctly extracted from the ConfigMap", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.VcPNID).To(Equal(configMap.Data["VcPNID"])) + Expect(providerConfig.VcPort).To(Equal(configMap.Data["VcPort"])) + }) + + Context("when VcPNID is unset in configMap", func() { + It("return an error", func() { + delete(configMap.Data, "VcPNID") + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).To(HaveOccurred()) + Expect(providerConfig).To(BeNil()) + }) + }) + + Context("StorageClassRequired", func() { + It("StorageClassRequired is unset in configMap", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.StorageClassRequired).To(BeFalse()) + }) + + Context("StorageClassRequired is set in configMap", func() { + BeforeEach(func() { + providerConfigIn.StorageClassRequired = true + }) + + It("StorageClassRequired is true in config", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.StorageClassRequired).To(BeTrue()) + }) + }) + }) + + Describe("Tests for TLS configuration", func() { + + Context("when no TLS configuration is specified", func() { + It("defaults to using TLS with the system root CA", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).NotTo(HaveOccurred()) + + Expect(providerConfig.InsecureSkipTLSVerify).To(BeFalse()) + Expect(providerConfig.CAFilePath).To(Equal("/etc/pki/tls/certs/ca-bundle.crt")) + }) + }) + + Context("when the config chooses to ignore TLS verification", func() { + BeforeEach(func() { + providerConfigIn.InsecureSkipTLSVerify = true + }) + + It("sets the insecure flag in the provider config", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).NotTo(HaveOccurred()) + Expect(providerConfig.InsecureSkipTLSVerify).To(BeTrue()) + }) + }) + + Context("when the TLS settings in the Config do not pars", func() { + It("returns an error when parsing the ConfigMa", func() { + configMap.Data["InsecureSkipTLSVerify"] = "bogus" + _, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when the config chooses to use TLS verification and overrides the CA file path", func() { + BeforeEach(func() { + providerConfigIn.CAFilePath = "/etc/a/new/ca/bundle.crt" + }) + + It("uses the new CA path", func() { + providerConfig, err := config.ConfigMapToProviderConfig(configMap, providerCreds) + Expect(err).ToNot(HaveOccurred()) + Expect(providerConfig.CAFilePath).To(Equal("/etc/a/new/ca/bundle.crt")) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/constants/constants.go b/pkg/vmprovider/providers/vsphere2/constants/constants.go new file mode 100644 index 000000000..ef26052b7 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/constants/constants.go @@ -0,0 +1,137 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package constants + +import ( + "github.com/vmware-tanzu/vm-operator/pkg" +) + +const ( + ExtraConfigTrue = "TRUE" + ExtraConfigFalse = "FALSE" + ExtraConfigUnset = "" + ExtraConfigGuestInfoPrefix = "guestinfo." + + // VCVMAnnotation Annotation placed on the VM. + VCVMAnnotation = "Virtual Machine managed by the vSphere Virtual Machine service" + + // ManagedByExtensionKey and ManagedByExtensionType represent the ManagedBy field on the VM. + // Historically, this field was used to differentiate VM Service managed VMs from traditional ones. + ManagedByExtensionKey = "com.vmware.vcenter.wcp" + ManagedByExtensionType = "VirtualMachine" + + // VSphereCustomizationBypassKey Annotation to skip applying VMware Tools Guest Customization. + VSphereCustomizationBypassKey = pkg.VMOperatorKey + "/vsphere-customization" + VSphereCustomizationBypassDisable = "disable" + + // VMOperatorV1Alpha1ExtraConfigKey Special ExtraConfig key for v1alpha1 images. + VMOperatorV1Alpha1ExtraConfigKey = "guestinfo.vmservice.defer-cloud-init" + VMOperatorV1Alpha1ConfigReady = "ready" + VMOperatorV1Alpha1ConfigEnabled = "enabled" + + // GOSCPendingExtraConfigKey and GOSCIgnoreToolsCheckExtraConfigKey are GOSC Related ExtraConfig keys. + GOSCPendingExtraConfigKey = "tools.deployPkg.fileName" + GOSCIgnoreToolsCheckExtraConfigKey = "vmware.tools.gosc.ignoretoolscheck" + + // EnableDiskUUIDExtraConfigKey Enable UUID ExtraConfig key. + EnableDiskUUIDExtraConfigKey = "disk.enableUUID" + + // MMPowerOffVMExtraConfigKey ExtraConfig key to enable DRS to powerOff VMs when underlying host enters into + // maintenance mode. This is to ensure the maintenance mode workflow is consistent for VMs with vGPU/DDPIO devices. + MMPowerOffVMExtraConfigKey = "maintenance.vm.evacuation.poweroff" + + // NetPlanVersion points to the version used for Network config. + // For more information, please see https://cloudinit.readthedocs.io/en/latest/topics/network-config-format-v2.html + NetPlanVersion = 2 + + // VMImageCLVersionAnnotation VirtualMachineImage annotation to cache the last fetched version. + VMImageCLVersionAnnotation = pkg.VMOperatorKey + "/content-library-version" + // VMImageCLVersionAnnotationVersion is the version of the VMImageCLVersionAnnotation for the VirtualMachineImage. + VMImageCLVersionAnnotationVersion = 1 + + PCIPassthruMMIOOverrideAnnotation = pkg.VMOperatorKey + "/pci-passthru-64bit-mmio-size" + PCIPassthruMMIOExtraConfigKey = "pciPassthru.use64bitMMIO" //nolint:gosec + PCIPassthruMMIOSizeExtraConfigKey = "pciPassthru.64bitMMIOSizeGB" //nolint:gosec + PCIPassthruMMIOSizeDefault = "512" + + // MinSupportedHWVersionForPVC is the supported virtual hardware version for persistent volumes. + MinSupportedHWVersionForPVC = 15 + // MinSupportedHWVersionForPCIPassthruDevices is the supported virtual hardware version for NVidia PCI devices. + MinSupportedHWVersionForPCIPassthruDevices = 17 + + // FirmwareOverrideAnnotation is the annotation key used for firmware override. + FirmwareOverrideAnnotation = pkg.VMOperatorKey + "/firmware" + + CloudInitTypeAnnotation = pkg.VMOperatorKey + "/cloudinit-type" + CloudInitTypeValueCloudInitPrep = "cloudinitprep" + CloudInitTypeValueGuestInfo = "guestinfo" + + CloudInitGuestInfoMetadata = "guestinfo.metadata" + CloudInitGuestInfoMetadataEncoding = "guestinfo.metadata.encoding" + CloudInitGuestInfoUserdata = "guestinfo.userdata" + CloudInitGuestInfoUserdataEncoding = "guestinfo.userdata.encoding" + + // InstanceStoragePVCNamePrefix prefix of auto-generated PVC names. + InstanceStoragePVCNamePrefix = "instance-pvc-" + // InstanceStorageLabelKey identifies resources related to instance storage. + // The primary purpose of this label is to identify instance storage resources such as + // PVCs and CNSNodeVMAttachments but not for List and kubectl-get of VM resources. + InstanceStorageLabelKey = "vmoperator.vmware.com/instance-storage-resource" + // InstanceStoragePVCsBoundAnnotationKey annotation key used to set bound state of all instance storage PVCs. + InstanceStoragePVCsBoundAnnotationKey = "vmoperator.vmware.com/instance-storage-pvcs-bound" + // InstanceStoragePVPlacementErrorAnnotationKey annotation key to set PV creation error. + // CSI reference to this annotation where it is defined: + // https://github.com/kubernetes-sigs/vsphere-csi-driver/blob/master/pkg/syncer/k8scloudoperator/placement.go + InstanceStoragePVPlacementErrorAnnotationKey = "failure-domain.beta.vmware.com/storagepool" + // InstanceStorageSelectedNodeMOIDAnnotationKey value corresponds to MOID of ESXi node that is elected to place instance storage volumes. + InstanceStorageSelectedNodeMOIDAnnotationKey = "vmoperator.vmware.com/instance-storage-selected-node-moid" + // InstanceStorageSelectedNodeAnnotationKey value corresponds to FQDN of ESXi node that is elected to place instance storage volumes. + InstanceStorageSelectedNodeAnnotationKey = "vmoperator.vmware.com/instance-storage-selected-node" + // KubernetesSelectedNodeAnnotationKey annotation key to set selected node on PVC. + KubernetesSelectedNodeAnnotationKey = "volume.kubernetes.io/selected-node" + // InstanceStoragePVPlacementErrorPrefix indicates prefix of error value. + InstanceStoragePVPlacementErrorPrefix = "FAILED_" + // InstanceStorageNotEnoughResErr is an error constant to indicate not enough resources. + InstanceStorageNotEnoughResErr = "FAILED_PLACEMENT-NotEnoughResources" + // InstanceStorageVDiskID vDisk ID for instance storage volume. + InstanceStorageVDiskID = "cc737f33-2aa3-4594-aa60-df7d6d4cb984" + + // XsiNamespace indicates the XML scheme instance namespace. + XsiNamespace = "http://www.w3.org/2001/XMLSchema-instance" + // ConfigSpecProviderXML indicates XML as the config spec transport type for virtual machine deployment. + ConfigSpecProviderXML = "XML" + + // V1alpha1FirstIP is an alias for versioned templating function V1alpha1_FirstIP. + V1alpha1FirstIP = "V1alpha1_FirstIP" + // V1alpha1FirstNicMacAddr is an alias for versioned templating function V1alpha1_FirstNicMacAddr. + V1alpha1FirstNicMacAddr = "V1alpha1_FirstNicMacAddr" + // V1alpha1FirstIPFromNIC is an alias for versioned templating function V1alpha1_FirstIPFromNIC. + V1alpha1FirstIPFromNIC = "V1alpha1_FirstIPFromNIC" + // V1alpha1IPsFromNIC is an alias for versioned templating function V1alpha1_IPsFromNIC. + V1alpha1IPsFromNIC = "V1alpha1_IPsFromNIC" + // V1alpha1FormatIP is an alias for versioned templating function V1alpha1_FormatIP. + V1alpha1FormatIP = "V1alpha1_FormatIP" + // V1alpha1IP is an alias for versioned templating function V1alpha1_IP. + V1alpha1IP = "V1alpha1_IP" + // V1alpha1SubnetMask is an alias for versioned templating function V1alpha1_SubnetMask. + V1alpha1SubnetMask = "V1alpha1_SubnetMask" + // V1alpha1FormatNameservers is an alias for versioned templating function V1alpha1_FormatNameservers. + V1alpha1FormatNameservers = "V1alpha1_FormatNameservers" + // V1alpha2FirstIP is an alias for versioned templating function V1alpha2_FirstIP. + V1alpha2FirstIP = "V1alpha2_FirstIP" + // V1alpha2FirstNicMacAddr is an alias for versioned templating function V1alpha2_FirstNicMacAddr. + V1alpha2FirstNicMacAddr = "V1alpha2_FirstNicMacAddr" + // V1alpha2FirstIPFromNIC is an alias for versioned templating function V1alpha2_FirstIPFromNIC. + V1alpha2FirstIPFromNIC = "V1alpha2_FirstIPFromNIC" + // V1alpha2IPsFromNIC is an alias for versioned templating function V1alpha2_IPsFromNIC. + V1alpha2IPsFromNIC = "V1alpha2_IPsFromNIC" + // V1alpha2FormatIP is an alias for versioned templating function V1alpha2_FormatIP. + V1alpha2FormatIP = "V1alpha2_FormatIP" + // V1alpha2IP is an alias for versioned templating function V1alpha2_IP. + V1alpha2IP = "V1alpha2_IP" + // V1alpha2SubnetMask is an alias for versioned templating function V1alpha2_SubnetMask. + V1alpha2SubnetMask = "V1alpha2_SubnetMask" + // V1alpha2FormatNameservers is an alias for versioned templating function V1alpha2_FormatNameservers. + V1alpha2FormatNameservers = "V1alpha2_FormatNameservers" +) diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go new file mode 100644 index 000000000..7dc3b9f54 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library.go @@ -0,0 +1,10 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary + +import ( + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var log = logf.Log.WithName("vsphere").WithName("contentlibrary") diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go new file mode 100644 index 000000000..be2ff212d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go @@ -0,0 +1,379 @@ +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary + +import ( + "context" + "io" + "net/url" + "os" + "path/filepath" + "strconv" + "time" + + k8serrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25/soap" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +type Provider interface { + GetLibraryItems(ctx context.Context, clUUID string) ([]library.Item, error) + GetLibraryItem(ctx context.Context, libraryUUID, itemName string, + notFoundReturnErr bool) (*library.Item, error) + GetLibraryItemID(ctx context.Context, itemUUID string) (*library.Item, error) + ListLibraryItems(ctx context.Context, libraryUUID string) ([]string, error) + UpdateLibraryItem(ctx context.Context, itemID, newName string, newDescription *string) error + RetrieveOvfEnvelopeFromLibraryItem(ctx context.Context, item *library.Item) (*ovf.Envelope, error) + RetrieveOvfEnvelopeByLibraryItemID(ctx context.Context, itemID string) (*ovf.Envelope, error) + + // TODO: Testing only. Remove these from this file. + CreateLibraryItem(ctx context.Context, libraryItem library.Item, path string) error +} + +type provider struct { + libMgr *library.Manager + retryInterval time.Duration +} + +const ( + EnvContentLibAPIWaitSecs = "CONTENT_API_WAIT_SECS" // BMV: Investigate if setting this to 1 actually reduces the integration test time. + DefaultContentLibAPIWaitSecs = 5 +) + +func IsSupportedDeployType(t string) bool { + switch t { + case library.ItemTypeVMTX, library.ItemTypeOVF: + // Keep in sync with what cloneVMFromContentLibrary() handles. + return true + default: + return false + } +} + +func NewProvider(restClient *rest.Client) Provider { + waitSeconds, err := strconv.Atoi(os.Getenv(EnvContentLibAPIWaitSecs)) + if err != nil || waitSeconds < 1 { + waitSeconds = DefaultContentLibAPIWaitSecs + } + + return NewProviderWithWaitSec(restClient, waitSeconds) +} + +func NewProviderWithWaitSec(restClient *rest.Client, waitSeconds int) Provider { + return &provider{ + libMgr: library.NewManager(restClient), + retryInterval: time.Duration(waitSeconds) * time.Second, + } +} + +func (cs *provider) ListLibraryItems(ctx context.Context, libraryUUID string) ([]string, error) { + logger := log.WithValues("libraryUUID", libraryUUID) + itemList, err := cs.libMgr.ListLibraryItems(ctx, libraryUUID) + if err != nil { + if lib.IsNotFoundError(err) { + logger.Error(err, "cannot list items from content library that does not exist") + return nil, nil + } + return nil, err + } + return itemList, err +} + +func (cs *provider) GetLibraryItems(ctx context.Context, libraryUUID string) ([]library.Item, error) { + logger := log.WithValues("libraryUUID", libraryUUID) + itemList, err := cs.libMgr.ListLibraryItems(ctx, libraryUUID) + if err != nil { + if lib.IsNotFoundError(err) { + logger.Error(err, "cannot list items from content library that does not exist") + return nil, nil + } + return nil, err + } + + // best effort to get content library items. + resErrs := make([]error, 0) + items := make([]library.Item, 0) + for _, itemID := range itemList { + item, err := cs.libMgr.GetLibraryItem(ctx, itemID) + if err != nil { + resErrs = append(resErrs, err) + logger.Error(err, "get library item failed", "itemID", itemID) + continue + } + items = append(items, *item) + } + + return items, k8serrors.NewAggregate(resErrs) +} + +func (cs *provider) GetLibraryItem(ctx context.Context, libraryUUID, itemName string, + notFoundReturnErr bool) (*library.Item, error) { + itemIDs, err := cs.libMgr.FindLibraryItems(ctx, library.FindItem{LibraryID: libraryUUID, Name: itemName}) + if err != nil { + return nil, errors.Wrapf(err, "failed to find image: %s", itemName) + } + + if len(itemIDs) == 0 { + if notFoundReturnErr { + return nil, errors.Errorf("no library item named: %s", itemName) + } + return nil, nil + } + if len(itemIDs) != 1 { + return nil, errors.Errorf("multiple library items named: %s", itemName) + } + + item, err := cs.libMgr.GetLibraryItem(ctx, itemIDs[0]) + if err != nil { + return nil, errors.Wrapf(err, "failed to get library item: %s", itemName) + } + + return item, nil +} + +func (cs *provider) GetLibraryItemID(ctx context.Context, itemUUID string) (*library.Item, error) { + item, err := cs.libMgr.GetLibraryItem(ctx, itemUUID) + if err != nil { + return nil, errors.Wrapf(err, "failed to find image: %s", itemUUID) + } + + return item, nil +} + +// RetrieveOvfEnvelopeByLibraryItemID retrieves the OVF Envelope by the given library item ID. +func (cs *provider) RetrieveOvfEnvelopeByLibraryItemID(ctx context.Context, itemID string) (*ovf.Envelope, error) { + libItem, err := cs.libMgr.GetLibraryItem(ctx, itemID) + if err != nil { + return nil, err + } + + if libItem == nil || libItem.Type != library.ItemTypeOVF { + log.Error(nil, "empty or non OVF library item type, skipping", "itemID", itemID) + // No need to return the error here to avoid unnecessary reconciliation. + return nil, nil + } + + return cs.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem) +} + +func readerFromURL(ctx context.Context, c *rest.Client, url *url.URL) (io.ReadCloser, error) { + p := soap.DefaultDownload + readerStream, _, err := c.Download(ctx, url, &p) + if err != nil { + // Log message used by VMC LINT. Refer to before making changes + log.Error(err, "Error occurred when downloading file", "url", url) + return nil, err + } + + return readerStream, nil +} + +// RetrieveOvfEnvelopeFromLibraryItem downloads the supported file from content library. +// parses the downloaded ovf and returns the OVF Envelope descriptor for consumption. +func (cs *provider) RetrieveOvfEnvelopeFromLibraryItem(ctx context.Context, item *library.Item) (*ovf.Envelope, error) { + // Create a download session for the file referred to by item id. + sessionID, err := cs.libMgr.CreateLibraryItemDownloadSession(ctx, library.Session{LibraryItemID: item.ID}) + if err != nil { + return nil, err + } + + logger := log.WithValues("sessionID", sessionID, "itemID", item.ID, "itemName", item.Name) + logger.V(4).Info("download session for item created") + + defer func() { + if err := cs.libMgr.DeleteLibraryItemDownloadSession(ctx, sessionID); err != nil { + logger.Error(err, "Error deleting download session") + } + }() + + // Download ovf from the library item. + fileURL, err := cs.generateDownloadURLForLibraryItem(ctx, logger, sessionID, item) + if err != nil { + return nil, err + } + + downloadedFileContent, err := readerFromURL(ctx, cs.libMgr.Client, fileURL) + if err != nil { + logger.Error(err, "error downloading file from library item") + return nil, err + } + + logger.V(4).Info("downloaded library item") + defer func() { + _ = downloadedFileContent.Close() + }() + + envelope, err := ovf.Unmarshal(downloadedFileContent) + if err != nil { + logger.Error(err, "error parsing the OVF envelope") + return nil, nil + } + + return envelope, nil +} + +// UpdateLibraryItem updates the content library item's name and description. +func (cs *provider) UpdateLibraryItem(ctx context.Context, itemID, newName string, newDescription *string) error { + log.Info("Updating Library Item", "itemID", itemID, + "newName", newName, "newDescription", newDescription) + + item, err := cs.libMgr.GetLibraryItem(ctx, itemID) + if err != nil { + log.Error(err, "error getting library item") + return err + } + + if newName != "" { + item.Name = newName + } + if newDescription != nil { + item.Description = newDescription + } + + return cs.libMgr.UpdateLibraryItem(ctx, item) +} + +// Only used in testing. +func (cs *provider) CreateLibraryItem(ctx context.Context, libraryItem library.Item, path string) error { + log.Info("Creating Library Item", "item", libraryItem, "path", path) + + itemID, err := cs.libMgr.CreateLibraryItem(ctx, libraryItem) + if err != nil { + return err + } + + sessionID, err := cs.libMgr.CreateLibraryItemUpdateSession(ctx, library.Session{LibraryItemID: itemID}) + if err != nil { + return err + } + + // Update Library item with library file "ovf" + uploadFunc := func(c *rest.Client, path string) error { + f, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + defer func() { + _ = f.Close() + }() + + fi, err := f.Stat() + if err != nil { + return err + } + + info := library.UpdateFile{ + Name: filepath.Base(path), + SourceType: "PUSH", + Size: fi.Size(), + } + + update, err := cs.libMgr.AddLibraryItemFile(ctx, sessionID, info) + if err != nil { + return err + } + + u, err := url.Parse(update.UploadEndpoint.URI) + if err != nil { + return err + } + + p := soap.DefaultUpload + p.ContentLength = info.Size + + return c.Upload(ctx, f, u, &p) + } + + if err = uploadFunc(cs.libMgr.Client, path); err != nil { + return err + } + + return cs.libMgr.CompleteLibraryItemUpdateSession(ctx, sessionID) +} + +// generateDownloadURLForLibraryItem downloads the file from content library in 3 steps: +// 1. list the available files and downloads only the ovf files based on filename suffix +// 2. prepare the download session and fetch the url to be used for download +// 3. download the file. +func (cs *provider) generateDownloadURLForLibraryItem( + ctx context.Context, + logger logr.Logger, + sessionID string, + item *library.Item) (*url.URL, error) { + + // List the files available for download in the library item. + files, err := cs.libMgr.ListLibraryItemDownloadSessionFile(ctx, sessionID) + if err != nil { + return nil, err + } + + var fileToDownload string + for _, file := range files { + logger.V(4).Info("Library Item file", "fileName", file.Name) + if ext := filepath.Ext(file.Name); ext != "" && IsSupportedDeployType(ext[1:]) { + fileToDownload = file.Name + break + } + } + if fileToDownload == "" { + return nil, errors.Errorf("No files with supported deploy type are available for download for %s", item.ID) + } + + _, err = cs.libMgr.PrepareLibraryItemDownloadSessionFile(ctx, sessionID, fileToDownload) + if err != nil { + return nil, err + } + + logger.V(4).Info("request posted to prepare file", "fileToDownload", fileToDownload) + + // Content library api to prepare a file for download guarantees eventual end state of either + // ERROR or PREPARED in order to avoid posting too many requests to the api. + var fileURL string + err = wait.PollUntilContextCancel(ctx, cs.retryInterval, true, func(_ context.Context) (bool, error) { + downloadSessResp, err := cs.libMgr.GetLibraryItemDownloadSession(ctx, sessionID) + if err != nil { + return false, err + } + + if downloadSessResp.ErrorMessage != nil { + return false, downloadSessResp.ErrorMessage + } + + info, err := cs.libMgr.GetLibraryItemDownloadSessionFile(ctx, sessionID, fileToDownload) + if err != nil { + return false, err + } + + if info.Status == "ERROR" { + // Log message used by VMC LINT. Refer to before making changes + return false, errors.Errorf("Error occurred preparing file for download %v", info.ErrorMessage) + } + + if info.Status != "PREPARED" { + return false, nil + } + + if info.DownloadEndpoint == nil { + return false, errors.Errorf("Prepared file for download does not have endpoint") + } + + fileURL = info.DownloadEndpoint.URI + log.V(4).Info("Downloaded file", "fileURL", fileURL) + return true, nil + }) + + if err != nil { + return nil, err + } + + return url.Parse(fileURL) +} diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go new file mode 100644 index 000000000..2055b76eb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("ContentLibrary Provider", clTests) +} + +var suite = builder.NewTestSuite() + +func TestContentLibrary(t *testing.T) { + suite.Register(t, "vSphere Provider ContentLibrary Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go new file mode 100644 index 000000000..968eac039 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go @@ -0,0 +1,139 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary_test + +import ( + "os" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vapi/library" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func clTests() { + Describe("Content Library", func() { + + var ( + initObjects []client.Object + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + + clProvider contentlibrary.Provider + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + testConfig.WithContentLibrary = true + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + clProvider = contentlibrary.NewProvider(ctx.RestClient) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + }) + + Context("when items are present in library", func() { + + It("List items id in library", func() { + items, err := clProvider.GetLibraryItems(ctx, ctx.ContentLibraryID) + Expect(err).ToNot(HaveOccurred()) + Expect(items).ToNot(BeEmpty()) + }) + + It("Does not return error when library does not exist", func() { + items, err := clProvider.GetLibraryItems(ctx, "dummy-cl") + Expect(err).ToNot(HaveOccurred()) + Expect(items).To(BeEmpty()) + }) + + It("Does not return error when item name is invalid when notFoundReturnErr is set to false", func() { + item, err := clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, "dummy-name", true) + Expect(err).To(HaveOccurred()) + Expect(item).To(BeNil()) + + item, err = clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, "dummy-name", false) + Expect(err).NotTo(HaveOccurred()) + Expect(item).To(BeNil()) + }) + + It("Gets items and returns OVF", func() { + item, err := clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, ctx.ContentLibraryImageName, true) + Expect(err).ToNot(HaveOccurred()) + Expect(item).ToNot(BeNil()) + + ovfEnvelope, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, item) + Expect(err).ToNot(HaveOccurred()) + Expect(ovfEnvelope).ToNot(BeNil()) + }) + }) + + Context("when items are not present in library", func() { + + Context("when invalid item id is passed", func() { + + It("returns an error creating a download session", func() { + libItem := &library.Item{ + Name: "fakeItem", + Type: "ovf", + LibraryID: "fakeID", + } + + ovf, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("404 Not Found")) + Expect(ovf).To(BeNil()) + }) + }) + }) + + Context("called with an OVF that is invalid", func() { + var ovfPath string + + AfterEach(func() { + if ovfPath != "" { + Expect(os.Remove(ovfPath)).To(Succeed()) + } + }) + + It("does not return error", func() { + ovf, err := os.CreateTemp("", "fake-*.ovf") + Expect(err).NotTo(HaveOccurred()) + ovfPath = ovf.Name() + + ovfInfo, err := ovf.Stat() + Expect(err).NotTo(HaveOccurred()) + + libItemName := strings.Split(ovfInfo.Name(), ".ovf")[0] + libItem := library.Item{ + Name: libItemName, + Type: "ovf", + LibraryID: ctx.ContentLibraryID, + } + + err = clProvider.CreateLibraryItem(ctx, libItem, ovfPath) + Expect(err).NotTo(HaveOccurred()) + + libItem2, err := clProvider.GetLibraryItem(ctx, ctx.ContentLibraryID, libItemName, true) + Expect(err).ToNot(HaveOccurred()) + Expect(libItem2).ToNot(BeNil()) + Expect(libItem2.Name).To(Equal(libItem.Name)) + + ovfEnvelope, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem2) + Expect(err).ToNot(HaveOccurred()) + Expect(ovfEnvelope).To(BeNil()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go new file mode 100644 index 000000000..b4a2fd866 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils.go @@ -0,0 +1,149 @@ +// Copyright (c) 2019-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary + +import ( + "regexp" + "strconv" + "strings" + + "github.com/vmware/govmomi/ovf" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +var vmxRe = regexp.MustCompile(`vmx-(\d+)`) + +// ParseVirtualHardwareVersion parses the virtual hardware version +// For eg. "vmx-15" returns 15. +func ParseVirtualHardwareVersion(vmxVersion string) int32 { + // obj is the full string and the submatch (\d+) and return a []string with values + obj := vmxRe.FindStringSubmatch(vmxVersion) + if len(obj) != 2 { + return 0 + } + + version, err := strconv.ParseInt(obj[1], 10, 32) + if err != nil { + return 0 + } + + return int32(version) +} + +// UpdateVmiWithOvfEnvelope updates the given vmi object with the content of given OVF envelope. +func UpdateVmiWithOvfEnvelope(vmi client.Object, ovfEnvelope ovf.Envelope) { + var status *vmopv1.VirtualMachineImageStatus + + switch vmi := vmi.(type) { + case *vmopv1.VirtualMachineImage: + status = &vmi.Status + case *vmopv1.ClusterVirtualMachineImage: + status = &vmi.Status + default: + return + } + + if ovfEnvelope.VirtualSystem != nil { + initImageStatusFromOVFVirtualSystem(status, ovfEnvelope.VirtualSystem) + + ovfSystemProps := getVmwareSystemPropertiesFromOvf(ovfEnvelope.VirtualSystem) + if len(ovfSystemProps) > 0 { + annotations := vmi.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + vmi.SetAnnotations(annotations) + } + + for k, v := range ovfSystemProps { + annotations[k] = v + } + } + } +} + +func initImageStatusFromOVFVirtualSystem( + imageStatus *vmopv1.VirtualMachineImageStatus, + ovfVirtualSystem *ovf.VirtualSystem) { + + // Use info from the first product section in the VM image, if one exists. + if product := ovfVirtualSystem.Product; len(product) > 0 { + p := product[0] + + productInfo := &imageStatus.ProductInfo + productInfo.Vendor = p.Vendor + productInfo.Product = p.Product + productInfo.Version = p.Version + productInfo.FullVersion = p.FullVersion + } + + // Use operating system info from the first os section in the VM image, if one exists. + if os := ovfVirtualSystem.OperatingSystem; len(os) > 0 { + o := os[0] + + osInfo := &imageStatus.OSInfo + osInfo.ID = strconv.Itoa(int(o.ID)) + if o.Version != nil { + osInfo.Version = *o.Version + } + if o.OSType != nil { + osInfo.Type = *o.OSType + } + } + + // Use hardware section info from the VM image, if one exists. + if virtualHW := ovfVirtualSystem.VirtualHardware; len(virtualHW) > 0 { + imageStatus.Firmware = getFirmwareType(virtualHW[0]) + + if sys := virtualHW[0].System; sys != nil && sys.VirtualSystemType != nil { + ver := ParseVirtualHardwareVersion(*sys.VirtualSystemType) + if ver != 0 { + imageStatus.HardwareVersion = &ver + } + } + } + + for _, product := range ovfVirtualSystem.Product { + for _, prop := range product.Property { + // Only show user configurable properties + if prop.UserConfigurable != nil && *prop.UserConfigurable { + property := vmopv1.OVFProperty{ + Key: prop.Key, + Type: prop.Type, + Default: prop.Default, + } + imageStatus.OVFProperties = append(imageStatus.OVFProperties, property) + } + } + } +} + +func getVmwareSystemPropertiesFromOvf(ovfVirtualSystem *ovf.VirtualSystem) map[string]string { + properties := make(map[string]string) + + if ovfVirtualSystem != nil { + for _, product := range ovfVirtualSystem.Product { + for _, prop := range product.Property { + if strings.HasPrefix(prop.Key, "vmware-system") { + if prop.Default != nil { + properties[prop.Key] = *prop.Default + } + } + } + } + } + + return properties +} + +// getFirmwareType returns the firmware type (eg: "efi", "bios") present in the virtual hardware section of the OVF. +func getFirmwareType(hardware ovf.VirtualHardwareSection) string { + for _, cfg := range hardware.Config { + if cfg.Key == "firmware" { + return cfg.Value + } + } + return "" +} diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go new file mode 100644 index 000000000..051f1bb91 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_utils_test.go @@ -0,0 +1,28 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package contentlibrary_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +var _ = Describe("ParseVirtualHardwareVersion", func() { + It("empty hardware string", func() { + vmxHwVersionString := "" + Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) + }) + + It("invalid hardware string", func() { + vmxHwVersionString := "blah" + Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) + }) + + It("valid hardware version string eg. vmx-15", func() { + vmxHwVersionString := "vmx-15" + Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(Equal(int32(15))) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/credentials/credentials.go b/pkg/vmprovider/providers/vsphere2/credentials/credentials.go new file mode 100644 index 000000000..9ac994bf5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/credentials/credentials.go @@ -0,0 +1,62 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials + +import ( + "context" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" +) + +// VSphereVMProviderCredentials wraps the data needed to login to vCenter. +type VSphereVMProviderCredentials struct { + Username string + Password string +} + +func GetProviderCredentials(client ctrlruntime.Client, namespace, secretName string) (*VSphereVMProviderCredentials, error) { + secret := &corev1.Secret{} + secretKey := types.NamespacedName{Namespace: namespace, Name: secretName} + if err := client.Get(context.Background(), secretKey, secret); err != nil { + // Log message used by VMC LINT. Refer to before making changes + return nil, errors.Wrapf(err, "cannot find secret for provider credentials: %s", secretKey) + } + + var credentials VSphereVMProviderCredentials + credentials.Username = string(secret.Data["username"]) + credentials.Password = string(secret.Data["password"]) + + if credentials.Username == "" || credentials.Password == "" { + return nil, errors.New("vCenter username and password are missing") + } + + return &credentials, nil +} + +func setSecretData(secret *corev1.Secret, credentials *VSphereVMProviderCredentials) { + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + + secret.Data["username"] = []byte(credentials.Username) + secret.Data["password"] = []byte(credentials.Password) +} + +// ProviderCredentialsToSecret returns the Secret for the credentials. +// Testing only. +func ProviderCredentialsToSecret(namespace string, credentials *VSphereVMProviderCredentials, vcCredsSecretName string) *corev1.Secret { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: vcCredsSecretName, + Namespace: namespace, + }, + } + setSecretData(secret, credentials) + + return secret +} diff --git a/pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go b/pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go new file mode 100644 index 000000000..45cedd1ea --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/credentials/credentials_suite_test.go @@ -0,0 +1,44 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/test" +) + +var ( + model *simulator.Model + server *simulator.Server + ctx context.Context + tlsTestModel *simulator.Model + tlsServer *simulator.Server + tlsServerCertPath string + tlsServerKeyPath string +) + +var _ = BeforeSuite(func() { + ctx, model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer = test.BeforeSuite() +}) + +var _ = AfterSuite(func() { + test.AfterSuite( + ctx, + model, server, + tlsServerKeyPath, tlsServerCertPath, + tlsTestModel, tlsServer) +}) + +func TestCredentials(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Credentials Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go b/pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go new file mode 100644 index 000000000..8950bdffa --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/credentials/credentials_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package credentials_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + + . "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/credentials" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func newSecret(name string, ns string, user string, pass string) (*corev1.Secret, *VSphereVMProviderCredentials) { + creds := &VSphereVMProviderCredentials{ + Username: user, + Password: pass, + } + secret := ProviderCredentialsToSecret(ns, creds, name) + return secret, creds +} + +var _ = Describe("GetProviderCredentials", func() { + + Context("when a good secret exists", func() { + Specify("returns good credentials with no error", func() { + secretIn, credsIn := newSecret("some-name", "some-namespace", "some-user", "some-pass") + client := builder.NewFakeClient(secretIn) + credsOut, err := GetProviderCredentials(client, secretIn.Namespace, secretIn.Name) + Expect(err).ToNot(HaveOccurred()) + Expect(credsOut).To(Equal(credsIn)) + }) + }) + + Context("when a bad secret exists", func() { + + Context("with empty username", func() { + Specify("returns no credentials with error", func() { + secretIn, _ := newSecret("some-name", "some-namespace", "", "some-pass") + client := builder.NewFakeClient(secretIn) + credsOut, err := GetProviderCredentials(client, secretIn.Namespace, secretIn.Name) + Expect(err).To(HaveOccurred()) + Expect(credsOut).To(BeNil()) + }) + }) + + Context("with empty password", func() { + Specify("returns no credentials with error", func() { + secretIn, _ := newSecret("some-name", "some-namespace", "some-user", "") + client := builder.NewFakeClient(secretIn) + credsOut, err := GetProviderCredentials(client, secretIn.Namespace, secretIn.Name) + Expect(err).To(HaveOccurred()) + Expect(credsOut).To(BeNil()) + }) + }) + }) + + Context("when no secret exists", func() { + Specify("returns no credentials with error", func() { + client := builder.NewFakeClient() + credsOut, err := GetProviderCredentials(client, "none-namespace", "none-name") + Expect(err).To(HaveOccurred()) + Expect(credsOut).To(BeNil()) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go b/pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go new file mode 100644 index 000000000..698faa7d5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/instancestorage/instance_storage.go @@ -0,0 +1,41 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package instancestorage + +import ( + "strings" + + apiErrors "k8s.io/apimachinery/pkg/api/errors" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +// IsPresent checks if VM Spec has instance volumes added to its Volumes list. +func IsPresent(vm *vmopv1.VirtualMachine) bool { + for _, vol := range vm.Spec.Volumes { + if pvc := vol.PersistentVolumeClaim; pvc != nil && pvc.InstanceVolumeClaim != nil { + return true + } + } + return false +} + +// FilterVolumes returns instance storage volumes present in VM spec. +func FilterVolumes(vm *vmopv1.VirtualMachine) []vmopv1.VirtualMachineVolume { + var volumes []vmopv1.VirtualMachineVolume + for _, vol := range vm.Spec.Volumes { + if pvc := vol.PersistentVolumeClaim; pvc != nil && pvc.InstanceVolumeClaim != nil { + volumes = append(volumes, vol) + } + } + + return volumes +} + +func IsInsufficientQuota(err error) bool { + if apiErrors.IsForbidden(err) && (strings.Contains(err.Error(), "insufficient quota") || strings.Contains(err.Error(), "exceeded quota")) { + return true + } + return false +} diff --git a/pkg/vmprovider/providers/vsphere2/internal/internal.go b/pkg/vmprovider/providers/vsphere2/internal/internal.go new file mode 100644 index 000000000..8a4f68da3 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/internal/internal.go @@ -0,0 +1,59 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//nolint:revive,stylecheck +package internal + +import ( + "context" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/soap" + "github.com/vmware/govmomi/vim25/types" +) + +type InvokeFSR_TaskRequest struct { + This types.ManagedObjectReference `xml:"_this"` +} + +type InvokeFSR_TaskResponse struct { + Returnval types.ManagedObjectReference `xml:"returnval"` +} + +type InvokeFSR_TaskBody struct { + Req *InvokeFSR_TaskRequest `xml:"urn:vim25 InvokeFSR_Task,omitempty"` + Res *InvokeFSR_TaskResponse `xml:"InvokeFSR_TaskResponse,omitempty"` + Fault_ *soap.Fault `xml:"http://schemas.xmlsoap.org/soap/envelope/ Fault,omitempty"` +} + +func (b *InvokeFSR_TaskBody) Fault() *soap.Fault { + return b.Fault_ +} + +func InvokeFSR_Task(ctx context.Context, r soap.RoundTripper, req *InvokeFSR_TaskRequest) (*InvokeFSR_TaskResponse, error) { + var reqBody, resBody InvokeFSR_TaskBody + reqBody.Req = req + if err := r.RoundTrip(ctx, &reqBody, &resBody); err != nil { + return nil, err + } + return resBody.Res, nil +} + +func VirtualMachineFSR(ctx context.Context, vm types.ManagedObjectReference, client *vim25.Client) (*object.Task, error) { + req := InvokeFSR_TaskRequest{ + This: vm, + } + res, err := InvokeFSR_Task(ctx, client, &req) + if err != nil { + return nil, err + } + return object.NewTask(client, res.Returnval), nil +} + +type CustomizationCloudinitPrep struct { + types.CustomizationIdentitySettings + + Metadata string `xml:"metadata"` + Userdata string `xml:"userdata,omitempty"` +} diff --git a/pkg/vmprovider/providers/vsphere2/network/gosc.go b/pkg/vmprovider/providers/vsphere2/network/gosc.go new file mode 100644 index 000000000..feb9f9f3a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/gosc.go @@ -0,0 +1,79 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network + +import ( + "net" + + vimtypes "github.com/vmware/govmomi/vim25/types" +) + +func GuestOSCustomization(results NetworkInterfaceResults) ([]vimtypes.CustomizationAdapterMapping, error) { + mappings := make([]vimtypes.CustomizationAdapterMapping, 0, len(results.Results)) + + for _, r := range results.Results { + adapter := vimtypes.CustomizationIPSettings{ + DnsServerList: r.Nameservers, + } + + if r.DHCP4 { + adapter.Ip = &vimtypes.CustomizationDhcpIpGenerator{} + } else { + // GOSC doesn't support multiple IPv4 address per interface so use the first one. Old code + // only ever set one gateway so do the same here too. + for _, ipConfig := range r.IPConfigs { + if !ipConfig.IsIPv4 { + continue + } + + ip, ipNet, err := net.ParseCIDR(ipConfig.IPCIDR) + if err != nil { + return nil, err + } + subnetMask := net.CIDRMask(ipNet.Mask.Size()) + + adapter.Ip = &vimtypes.CustomizationFixedIp{IpAddress: ip.String()} + adapter.SubnetMask = net.IP(subnetMask).String() + adapter.Gateway = []string{ipConfig.Gateway} + break + } + } + + if r.DHCP6 { + adapter.IpV6Spec = &vimtypes.CustomizationIPSettingsIpV6AddressSpec{ + Ip: []vimtypes.BaseCustomizationIpV6Generator{ + &vimtypes.CustomizationDhcpIpV6Generator{}, + }, + } + } else { + for _, ipConfig := range r.IPConfigs { + if ipConfig.IsIPv4 { + continue + } + + ip, ipNet, err := net.ParseCIDR(ipConfig.IPCIDR) + if err != nil { + return nil, err + } + ones, _ := ipNet.Mask.Size() + + if adapter.IpV6Spec == nil { + adapter.IpV6Spec = &vimtypes.CustomizationIPSettingsIpV6AddressSpec{} + } + adapter.IpV6Spec.Ip = append(adapter.IpV6Spec.Ip, &vimtypes.CustomizationFixedIpV6{ + IpAddress: ip.String(), + SubnetMask: int32(ones), + }) + adapter.IpV6Spec.Gateway = append(adapter.IpV6Spec.Gateway, ipConfig.Gateway) + } + } + + mappings = append(mappings, vimtypes.CustomizationAdapterMapping{ + MacAddress: r.MacAddress, + Adapter: adapter, + }) + } + + return mappings, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/network/gosc_test.go b/pkg/vmprovider/providers/vsphere2/network/gosc_test.go new file mode 100644 index 000000000..1eea859a8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/gosc_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +var _ = Describe("GOSC", func() { + const ( + macAddr1 = "50-8A-80-9D-28-22" + + ipv4Gateway = "192.168.1.1" + ipv4 = "192.168.1.10" + ipv4CIDR = ipv4 + "/24" + ipv6Gateway = "fd8e:b5a0:f172:123::1" + ipv6 = "fd8e:b5a0:f172:123::f" + ipv6Subnet = 48 + + dnsServer1 = "9.9.9.9" + ) + + Context("GuestOSCustomization", func() { + + var ( + results network.NetworkInterfaceResults + adapterMappings []types.CustomizationAdapterMapping + err error + ) + + BeforeEach(func() { + results.Results = nil + }) + + JustBeforeEach(func() { + adapterMappings, err = network.GuestOSCustomization(results) + }) + + Context("IPv4/6 Static adapter", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + IPCIDR: ipv4CIDR, + IsIPv4: true, + Gateway: ipv4Gateway, + }, + { + IPCIDR: ipv6 + fmt.Sprintf("/%d", ipv6Subnet), + IsIPv4: false, + Gateway: ipv6Gateway, + }, + }, + MacAddress: macAddr1, + Name: "eth0", + DHCP4: false, + DHCP6: false, + MTU: 1500, // AFAIK not supported via GOSC + Nameservers: []string{dnsServer1}, + Routes: nil, // AFAIK not supported via GOSC + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(adapterMappings).To(HaveLen(1)) + mapping := adapterMappings[0] + + adapter := mapping.Adapter + Expect(mapping.MacAddress).To(Equal(macAddr1)) + Expect(adapter.Gateway).To(Equal([]string{ipv4Gateway})) + Expect(adapter.SubnetMask).To(Equal("255.255.255.0")) + Expect(adapter.DnsServerList).To(Equal([]string{dnsServer1})) + Expect(adapter.Ip).To(BeAssignableToTypeOf(&types.CustomizationFixedIp{})) + fixedIP := adapter.Ip.(*types.CustomizationFixedIp) + Expect(fixedIP.IpAddress).To(Equal(ipv4)) + + ipv6Spec := adapter.IpV6Spec + Expect(ipv6Spec).ToNot(BeNil()) + Expect(ipv6Spec.Gateway).To(Equal([]string{ipv6Gateway})) + Expect(ipv6Spec.Ip).To(HaveLen(1)) + Expect(ipv6Spec.Ip[0]).To(BeAssignableToTypeOf(&types.CustomizationFixedIpV6{})) + addressSpec := ipv6Spec.Ip[0].(*types.CustomizationFixedIpV6) + Expect(addressSpec.IpAddress).To(Equal(ipv6)) + Expect(addressSpec.SubnetMask).To(BeEquivalentTo(ipv6Subnet)) + }) + }) + + Context("IPv4/6 DHCP", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr1, + Name: "eth0", + DHCP4: true, + DHCP6: true, + MTU: 1500, // AFAIK not support via GOSC + Nameservers: []string{dnsServer1}, + Routes: nil, // AFAIK not support via GOSC + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(adapterMappings).To(HaveLen(1)) + mapping := adapterMappings[0] + + adapter := mapping.Adapter + Expect(mapping.MacAddress).To(Equal(macAddr1)) + Expect(adapter.Gateway).To(BeEmpty()) + Expect(adapter.SubnetMask).To(BeEmpty()) + + Expect(adapter.Ip).To(BeAssignableToTypeOf(&types.CustomizationDhcpIpGenerator{})) + ipv6Spec := adapter.IpV6Spec + Expect(ipv6Spec).ToNot(BeNil()) + Expect(ipv6Spec.Ip).To(HaveLen(1)) + Expect(ipv6Spec.Ip[0]).To(BeAssignableToTypeOf(&types.CustomizationDhcpIpV6Generator{})) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/network/netplan.go b/pkg/vmprovider/providers/vsphere2/network/netplan.go new file mode 100644 index 000000000..71ca80b45 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/netplan.go @@ -0,0 +1,103 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network + +import ( + "strings" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +// Netplan representation described in https://via.vmw.com/cloud-init-netplan // FIXME: 404. +type Netplan struct { + Version int `yaml:"version,omitempty"` + Ethernets map[string]NetplanEthernet `yaml:"ethernets,omitempty"` +} + +type NetplanEthernet struct { + Match NetplanEthernetMatch `yaml:"match,omitempty"` + SetName string `yaml:"set-name,omitempty"` + Dhcp4 bool `yaml:"dhcp4,omitempty"` + Dhcp6 bool `yaml:"dhcp6,omitempty"` + Addresses []string `yaml:"addresses,omitempty"` + Gateway4 string `yaml:"gateway4,omitempty"` + Gateway6 string `yaml:"gateway6,omitempty"` + MTU int64 `yaml:"mtu,omitempty"` + Nameservers NetplanEthernetNameserver `yaml:"nameservers,omitempty"` + Routes []NetplanEthernetRoute `yaml:"routes,omitempty"` +} + +type NetplanEthernetMatch struct { + MacAddress string `yaml:"macaddress,omitempty"` +} + +type NetplanEthernetNameserver struct { + Addresses []string `yaml:"addresses,omitempty"` + Search []string `yaml:"search,omitempty"` +} + +type NetplanEthernetRoute struct { + To string `yaml:"to"` + Via string `yaml:"via"` + Metric int32 `yaml:"metric,omitempty"` +} + +func NetPlanCustomization(result NetworkInterfaceResults) (*Netplan, error) { + netPlan := &Netplan{ + Version: constants.NetPlanVersion, + Ethernets: make(map[string]NetplanEthernet), + } + + for _, r := range result.Results { + npEth := NetplanEthernet{ + Match: NetplanEthernetMatch{ + MacAddress: NormalizeNetplanMac(r.MacAddress), + }, + SetName: r.Name, + MTU: r.MTU, + Nameservers: NetplanEthernetNameserver{ + Addresses: r.Nameservers, + Search: r.SearchDomains, + }, + } + + npEth.Dhcp4 = r.DHCP4 + npEth.Dhcp6 = r.DHCP6 + + if !npEth.Dhcp4 { + for _, ipConfig := range r.IPConfigs { + if ipConfig.IsIPv4 { + if npEth.Gateway4 == "" { + npEth.Gateway4 = ipConfig.Gateway + } + npEth.Addresses = append(npEth.Addresses, ipConfig.IPCIDR) + } + } + } + if !npEth.Dhcp6 { + for _, ipConfig := range r.IPConfigs { + if !ipConfig.IsIPv4 { + if npEth.Gateway6 == "" { + npEth.Gateway6 = ipConfig.Gateway + } + npEth.Addresses = append(npEth.Addresses, ipConfig.IPCIDR) + } + } + } + + for _, route := range r.Routes { + npEth.Routes = append(npEth.Routes, NetplanEthernetRoute(route)) + } + + netPlan.Ethernets[npEth.SetName] = npEth + } + + return netPlan, nil +} + +// NormalizeNetplanMac normalizes the mac address format to one compatible with netplan. +func NormalizeNetplanMac(mac string) string { + mac = strings.ReplaceAll(mac, "-", ":") + return strings.ToLower(mac) +} diff --git a/pkg/vmprovider/providers/vsphere2/network/netplan_test.go b/pkg/vmprovider/providers/vsphere2/network/netplan_test.go new file mode 100644 index 000000000..5e3e68375 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/netplan_test.go @@ -0,0 +1,143 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +var _ = Describe("Netplan", func() { + const ( + ifName = "eth0" + macAddr1 = "50-8A-80-9D-28-22" + macAddr1Norm = "50:8a:80:9d:28:22" + ipv4Gateway = "192.168.1.1" + ipv4 = "192.168.1.10" + ipv4CIDR = ipv4 + "/24" + ipv6Gateway = "fd8e:b5a0:f172:123::1" + ipv6 = "fd8e:b5a0:f172:123::f" + ipv6Subnet = 48 + dnsServer1 = "9.9.9.9" + searchDomain1 = "foobar.local" + ) + + Context("NetPlanCustomization", func() { + + var ( + results network.NetworkInterfaceResults + netplan *network.Netplan + err error + ) + + BeforeEach(func() { + results.Results = nil + netplan = nil + }) + + JustBeforeEach(func() { + netplan, err = network.NetPlanCustomization(results) + }) + + Context("IPv4/6 Static adapter", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + IPCIDR: ipv4CIDR, + IsIPv4: true, + Gateway: ipv4Gateway, + }, + { + IPCIDR: ipv6 + fmt.Sprintf("/%d", ipv6Subnet), + IsIPv4: false, + Gateway: ipv6Gateway, + }, + }, + MacAddress: macAddr1, + Name: ifName, + DHCP4: false, + DHCP6: false, + MTU: 1500, + Nameservers: []string{dnsServer1}, + SearchDomains: []string{searchDomain1}, + Routes: []network.NetworkInterfaceRoute{ + { + To: "185.107.56.59", + Via: "10.1.1.1", + Metric: 42, + }, + }, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(netplan).ToNot(BeNil()) + Expect(netplan.Version).To(Equal(constants.NetPlanVersion)) + + Expect(netplan.Ethernets).To(HaveLen(1)) + Expect(netplan.Ethernets).To(HaveKey(ifName)) + + np := netplan.Ethernets[ifName] + Expect(np.Match.MacAddress).To(Equal(macAddr1Norm)) + Expect(np.SetName).To(Equal(ifName)) + Expect(np.Dhcp4).To(BeFalse()) + Expect(np.Dhcp6).To(BeFalse()) + Expect(np.Gateway4).To(Equal(ipv4Gateway)) + Expect(np.Gateway6).To(Equal(ipv6Gateway)) + Expect(np.MTU).To(BeEquivalentTo(1500)) + Expect(np.Nameservers.Addresses).To(Equal([]string{dnsServer1})) + Expect(np.Nameservers.Search).To(Equal([]string{searchDomain1})) + Expect(np.Routes).To(HaveLen(1)) + route := np.Routes[0] + Expect(route.To).To(Equal("185.107.56.59")) + Expect(route.Via).To(Equal("10.1.1.1")) + Expect(route.Metric).To(BeEquivalentTo(42)) + }) + }) + + Context("IPv4/6 DHCP", func() { + BeforeEach(func() { + results.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr1, + Name: "eth0", + DHCP4: true, + DHCP6: true, + MTU: 9000, + Nameservers: []string{dnsServer1}, + SearchDomains: []string{searchDomain1}, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(netplan).ToNot(BeNil()) + Expect(netplan.Version).To(Equal(constants.NetPlanVersion)) + + Expect(netplan.Ethernets).To(HaveLen(1)) + Expect(netplan.Ethernets).To(HaveKey(ifName)) + + np := netplan.Ethernets[ifName] + Expect(np.Match.MacAddress).To(Equal(macAddr1Norm)) + Expect(np.SetName).To(Equal(ifName)) + Expect(np.Dhcp4).To(BeTrue()) + Expect(np.Dhcp6).To(BeTrue()) + Expect(np.MTU).To(BeEquivalentTo(9000)) + Expect(np.Nameservers.Addresses).To(Equal([]string{dnsServer1})) + Expect(np.Nameservers.Search).To(Equal([]string{searchDomain1})) + Expect(np.Routes).To(BeEmpty()) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/network/network.go b/pkg/vmprovider/providers/vsphere2/network/network.go new file mode 100644 index 000000000..e5d5d96e5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/network.go @@ -0,0 +1,630 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//nolint:revive +package network + +import ( + goctx "context" + "fmt" + "net" + "strings" + "time" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + vimtypes "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" +) + +type NetworkInterfaceResults struct { + Results []NetworkInterfaceResult +} + +type NetworkInterfaceResult struct { + IPConfigs []NetworkInterfaceIPConfig + MacAddress string + ExternalID string + NetworkID string + Backing object.NetworkReference + + Device vimtypes.BaseVirtualDevice + + // Fields from the InterfaceSpec used later during customization. + Name string + DHCP4 bool + DHCP6 bool + MTU int64 + Nameservers []string + SearchDomains []string + Routes []NetworkInterfaceRoute +} + +type NetworkInterfaceIPConfig struct { + IPCIDR string // IP address in CIDR notation e.g. 192.168.10.42/24 + IsIPv4 bool + Gateway string +} + +type NetworkInterfaceRoute struct { + To string + Via string + Metric int32 +} + +const ( + retryInterval = 100 * time.Millisecond + defaultEthernetCardType = "vmxnet3" +) + +var ( + // RetryTimeout is var so tests can change it to shorten tests until we get rid of the poll. + RetryTimeout = 15 * time.Second +) + +// CreateAndWaitForNetworkInterfaces creates the appropriate CRs for the VM's network +// interfaces, and then waits for them to be reconciled by NCP (NSX-T) or NetOP (VDS). +// +// Networking has always been kind of a pain and clunky for us, and unfortunately this +// code suffers gotchas and other not-so-great limitations. +// +// - Historically, this code used wait.PollImmediate() and we continue to do so here, +// but eventually we should Watch() these resources. Note though, that in the very +// common case, the CR is reconciled before our poll timeout, so that does save us +// from bailing out of the Reconcile. +// - Both NCP and NetOP CR Status inform us of the backing and IPAM info. However, for +// our InterfaceSpec we allow for DHCP but neither NCP nor NetOP has a way for us to +// mark the CR to don't do IPAM or to check DHCP is even enabled on the network. So +// this burns an IP, and the user must know that DHCP is actually configured. +// - CR naming has mostly been working by luck, and sometimes didn't offer very good +// discoverability. Here, with v1a2 we now have a "name" field in our InterfaceSpec, +// so we use that (BMV: need to double-check that field meets k8s name requirements) +// A longer term option is to use GenerateName to ensure a unique name, and then +// client.List() and filter by the OwnerRef to find the VM's network CRs, and to +// annotate the CRs to help identify which VM InterfaceSpec it corresponds to. +// Note that for existing v1a1 VMs we may need to add legacy name support here to +// find their interface CRs. +// - Instead of CreateOrUpdate, use CreateOrPatch to lessen the odds of blowing away +// any new fields. +func CreateAndWaitForNetworkInterfaces( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + vimClient *vim25.Client, + finder *find.Finder, + clusterMoRef *vimtypes.ManagedObjectReference, + interfaces []vmopv1.VirtualMachineNetworkInterfaceSpec) (NetworkInterfaceResults, error) { + + networkType := lib.GetNetworkProviderType() + if networkType == "" { + return NetworkInterfaceResults{}, fmt.Errorf("no network provider set") + } + + results := make([]NetworkInterfaceResult, 0, len(interfaces)) + + for i := range interfaces { + interfaceSpec := &interfaces[i] + + var result *NetworkInterfaceResult + var err error + + switch networkType { + case lib.NetworkProviderTypeVDS: + result, err = createNetOPNetworkInterface(vmCtx, client, vimClient, interfaceSpec) + case lib.NetworkProviderTypeNSXT: + result, err = createNCPNetworkInterface(vmCtx, client, vimClient, clusterMoRef, interfaceSpec) + case lib.NetworkProviderTypeNamed: + result, err = createNamedNetworkInterface(vmCtx, finder, interfaceSpec) + default: + err = fmt.Errorf("unsupported network provider envvar value: %q", networkType) + } + + if err != nil { + return NetworkInterfaceResults{}, + fmt.Errorf("network interface %q error: %w", interfaceSpec.Name, err) + } + + applyInterfaceSpecToResult(interfaceSpec, result) + results = append(results, *result) + } + + // TODO: Once we really support network changing on the fly, we need to keep track of now + // unused network interface CRDs so they can be deleted after they're removed from the VM + // via Reconfigure, instead of delaying that until the VM is deleted via GC. + + return NetworkInterfaceResults{ + Results: results, + }, nil +} + +// applyInterfaceSpecToResult applies the InterfaceSpec to results. Much of the InterfaceSpec - like DHCP - +// cannot be specified to the underlying network provider so apply those overrides to the results. +func applyInterfaceSpecToResult( + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec, + result *NetworkInterfaceResult) { + + // We don't really support IPv6 yet so don't enable it when the underlying provider didn't return any IPs. + dhcp4 := interfaceSpec.DHCP4 || len(result.IPConfigs) == 0 + dhcp6 := interfaceSpec.DHCP6 + + if len(interfaceSpec.Addresses) > 0 { + // The InterfaceSpec takes precedence over what underlying network provider says, so in this case it + // likely it did IPAM but we'll ignore those IPs. Providing static IPs via the Addresses field is + // probably not very common so override the IPConfigs so bootstrap has only one field to use. + result.IPConfigs = make([]NetworkInterfaceIPConfig, 0, len(interfaceSpec.Addresses)) + + for _, addr := range interfaceSpec.Addresses { + ip, _, err := net.ParseCIDR(addr) + if err != nil { + continue + } + + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: addr, + IsIPv4: ip.To4() != nil, + } + + if ipConfig.IsIPv4 { + dhcp4 = false + ipConfig.Gateway = interfaceSpec.Gateway4 + } else { + dhcp6 = false + ipConfig.Gateway = interfaceSpec.Gateway6 + } + + result.IPConfigs = append(result.IPConfigs, ipConfig) + } + } + + result.Name = interfaceSpec.Name + result.DHCP4 = dhcp4 + result.DHCP6 = dhcp6 + result.Nameservers = interfaceSpec.Nameservers + result.SearchDomains = interfaceSpec.SearchDomains + + if interfaceSpec.MTU != nil { + result.MTU = *interfaceSpec.MTU + } + for _, route := range interfaceSpec.Routes { + result.Routes = append(result.Routes, NetworkInterfaceRoute{To: route.To, Via: route.Via, Metric: route.Metric}) + } +} + +func createNamedNetworkInterface( + vmCtx context.VirtualMachineContextA2, + finder *find.Finder, + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec) (*NetworkInterfaceResult, error) { + + if interfaceSpec.Network.Kind != "" || interfaceSpec.Network.APIVersion != "" { + return nil, fmt.Errorf("network TypeMeta not supported for name network: %v", interfaceSpec.Network.TypeMeta) + } + + networkName := interfaceSpec.Network.Name + if networkName == "" { + return nil, fmt.Errorf("network name is required") + } + + backing, err := finder.Network(vmCtx, networkName) + if err != nil { + return nil, fmt.Errorf("unable to find named network %q: %w", networkName, err) + } + + return &NetworkInterfaceResult{ + NetworkID: networkName, + Backing: backing, + }, nil +} + +// NetOPCRName returns the name to be used for the NetOP NetworkInterface CR. +func NetOPCRName(vmName, networkName, interfaceName string, isV1A1 bool) string { + var name string + + if isV1A1 { + // Old naming convention: each network can really only have 1 NIC. + if networkName != "" { + name = fmt.Sprintf("%s-%s", networkName, vmName) + } else { + name = vmName + } + } else { + if networkName != "" { + name = fmt.Sprintf("%s-%s-%s", vmName, networkName, interfaceName) + } else { + name = fmt.Sprintf("%s-%s", vmName, interfaceName) + } + } + + return name +} + +func createNetOPNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + vimClient *vim25.Client, + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec) (*NetworkInterfaceResult, error) { + + if kind := interfaceSpec.Network.Kind; kind != "" && kind != "Network" { + return nil, fmt.Errorf("network kind %q is not supported for VDS", kind) + } + + // If empty, NetOP will try to select a namespace default. + networkName := interfaceSpec.Network.Name + + netIf := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: NetOPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, false), + Namespace: vmCtx.VM.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(vmCtx, client, netIf, func() error { + if err := controllerutil.SetOwnerReference(vmCtx.VM, netIf, client.Scheme()); err != nil { + // If this fails we likely have an object name collision, and we're in a tough spot. + return err + } + + netIf.Spec.NetworkName = networkName + // NetOP only defines a VMXNet3 type, but it doesn't really matter for our purposes. + netIf.Spec.Type = netopv1alpha1.NetworkInterfaceTypeVMXNet3 + return nil + }) + + if err != nil { + return nil, err + } + + netIf, err = waitForReadyNetworkInterface(vmCtx, client, netIf.Name) + if err != nil { + return nil, err + } + + return netOpNetIfToResult(vimClient, netIf), nil +} + +func netOpNetIfToResult( + vimClient *vim25.Client, + netIf *netopv1alpha1.NetworkInterface) *NetworkInterfaceResult { + + ipConfigs := make([]NetworkInterfaceIPConfig, 0, len(netIf.Status.IPConfigs)) + for _, ip := range netIf.Status.IPConfigs { + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: ipCIDRNotation(ip.IP, ip.SubnetMask, ip.IPFamily == netopv1alpha1.IPv4Protocol), + IsIPv4: ip.IPFamily == netopv1alpha1.IPv4Protocol, + Gateway: ip.Gateway, + } + ipConfigs = append(ipConfigs, ipConfig) + } + + pgObjRef := vimtypes.ManagedObjectReference{ + Type: "DistributedVirtualPortgroup", + Value: netIf.Status.NetworkID, + } + + return &NetworkInterfaceResult{ + IPConfigs: ipConfigs, + MacAddress: netIf.Status.MacAddress, // Not set by NetOP. + ExternalID: netIf.Status.ExternalID, // Ditto. + NetworkID: netIf.Status.NetworkID, + Backing: object.NewDistributedVirtualPortgroup(vimClient, pgObjRef), + } +} + +func findNetOPCondition( + netIf *netopv1alpha1.NetworkInterface, + condType netopv1alpha1.NetworkInterfaceConditionType) *netopv1alpha1.NetworkInterfaceCondition { + + for i := range netIf.Status.Conditions { + if netIf.Status.Conditions[i].Type == condType { + return &netIf.Status.Conditions[i] + } + } + return nil +} + +func waitForReadyNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + name string) (*netopv1alpha1.NetworkInterface, error) { + + netIf := &netopv1alpha1.NetworkInterface{} + netIfKey := types.NamespacedName{Namespace: vmCtx.VM.Namespace, Name: name} + + // TODO: Watch() this type instead. + err := wait.PollUntilContextTimeout(vmCtx, retryInterval, RetryTimeout, true, func(_ goctx.Context) (bool, error) { + if err := client.Get(vmCtx, netIfKey, netIf); err != nil { + return false, ctrlruntime.IgnoreNotFound(err) + } + + cond := findNetOPCondition(netIf, netopv1alpha1.NetworkInterfaceReady) + return cond != nil && cond.Status == corev1.ConditionTrue, nil + }) + + if err != nil { + if wait.Interrupted(err) { + // Try to return a more meaningful error when timed out. + if cond := findNetOPCondition(netIf, netopv1alpha1.NetworkInterfaceFailure); cond != nil && cond.Status == corev1.ConditionTrue { + return nil, fmt.Errorf("network interface failure: %s - %s", cond.Reason, cond.Message) + } + if cond := findNetOPCondition(netIf, netopv1alpha1.NetworkInterfaceReady); cond != nil && cond.Status == corev1.ConditionFalse { + return nil, fmt.Errorf("network interface is not ready: %s - %s", cond.Reason, cond.Message) + } + return nil, fmt.Errorf("network interface is not ready yet") + } + + return nil, err + } + + return netIf, nil +} + +// NCPCRName returns the name to be used for the NCP VirtualNetworkInterface CR. +func NCPCRName(vmName, networkName, interfaceName string, isV1A1 bool) string { + var name string + + if isV1A1 { + name = fmt.Sprintf("%s-lsp", vmName) + if networkName != "" { + name = fmt.Sprintf("%s-%s", networkName, name) + } + + } else { + if networkName != "" { + name = fmt.Sprintf("%s-%s-%s", vmName, networkName, interfaceName) + } else { + name = fmt.Sprintf("%s-%s", vmName, interfaceName) + } + } + + return name +} + +func createNCPNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + vimClient *vim25.Client, + clusterMoRef *vimtypes.ManagedObjectReference, + interfaceSpec *vmopv1.VirtualMachineNetworkInterfaceSpec) (*NetworkInterfaceResult, error) { + + // TODO: Do we need to still support the odd-ball NetOP in NSX-T? Sigh. Do that check here if needed. + if kind := interfaceSpec.Network.Kind; kind != "" && kind != "VirtualNetwork" { + return nil, fmt.Errorf("network kind %q is not supported for NCP", kind) + } + + // If empty, NCP will use the namespace default. + networkName := interfaceSpec.Network.Name + + vnetIf := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: NCPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, false), + Namespace: vmCtx.VM.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(vmCtx, client, vnetIf, func() error { + if err := controllerutil.SetOwnerReference(vmCtx.VM, vnetIf, client.Scheme()); err != nil { + return err + } + + vnetIf.Spec.VirtualNetwork = networkName + return nil + }) + + if err != nil { + return nil, err + } + + vnetIf, err = waitForReadyNCPNetworkInterface(vmCtx, client, vnetIf.Name) + if err != nil { + return nil, err + } + + return ncpNetIfToResult(vmCtx, vimClient, clusterMoRef, vnetIf) +} + +func ncpNetIfToResult( + ctx goctx.Context, + vimClient *vim25.Client, + clusterMoRef *vimtypes.ManagedObjectReference, + vnetIf *ncpv1alpha1.VirtualNetworkInterface) (*NetworkInterfaceResult, error) { + + // NSX-T makes the backing determination difficult: NsxLogicalSwitchID must be mapped to an + // actual DVPG since that is the backing, but the DVPG can, in some very rare but supported + // configurations, vary between CCRs. If we know the CCR - ether the VM already exists, or + // (for later work) we might pre-determine CCR w/o placement if there is only one possibility - + // get that backing now. + // Otherwise, we'll do it post-placement via ResolveNCPBackingPostPlacement() so that we create + // the VM with the correct backing. That means we cannot make this a part of the PlaceVMxCluster() + // ConfigSpec since we don't know the backing: we'd have to pre-filter the placement candidates. + // What a mess. This is an unfortunate decision that forces mapping logic to every NCP consumer. + + var backing object.NetworkReference + networkID := vnetIf.Status.ProviderStatus.NsxLogicalSwitchID + + if clusterMoRef != nil { + ccr := object.NewClusterComputeResource(vimClient, *clusterMoRef) + + networkRef, err := searchNsxtNetworkReference(ctx, ccr, networkID) + if err != nil { + return nil, err + } + + backing = networkRef + } + + var ipConfigs []NetworkInterfaceIPConfig + if ipAddress := vnetIf.Status.IPAddresses; len(ipAddress) == 0 || (len(ipAddress) == 1 && ipAddress[0].IP == "") { + // NCP's way of saying DHCP. + } else { + // Historically, we only grabbed the first entry and assume it is always IPv4 (!!!). Try to do slightly better. + for _, ipAddr := range ipAddress { + if ipAddr.IP == "" { + continue + } + + isIPv4 := net.ParseIP(ipAddr.IP).To4() != nil + ipConfig := NetworkInterfaceIPConfig{ + IPCIDR: ipCIDRNotation(ipAddr.IP, ipAddr.SubnetMask, isIPv4), + IsIPv4: isIPv4, + Gateway: ipAddr.Gateway, + } + + ipConfigs = append(ipConfigs, ipConfig) + } + } + + result := &NetworkInterfaceResult{ + IPConfigs: ipConfigs, + MacAddress: vnetIf.Status.MacAddress, + ExternalID: vnetIf.Status.InterfaceID, + NetworkID: networkID, + Backing: backing, + } + + return result, nil +} + +func waitForReadyNCPNetworkInterface( + vmCtx context.VirtualMachineContextA2, + client ctrlruntime.Client, + name string) (*ncpv1alpha1.VirtualNetworkInterface, error) { + + vnetIf := &ncpv1alpha1.VirtualNetworkInterface{} + vnetIfKey := types.NamespacedName{Namespace: vmCtx.VM.Namespace, Name: name} + + // TODO: Watch() this type instead. + err := wait.PollUntilContextTimeout(vmCtx, retryInterval, RetryTimeout, true, func(_ goctx.Context) (bool, error) { + if err := client.Get(vmCtx, vnetIfKey, vnetIf); err != nil { + return false, ctrlruntime.IgnoreNotFound(err) + } + + for _, condition := range vnetIf.Status.Conditions { + // TODO: Does NCP define condition constants? + if strings.Contains(condition.Type, "Ready") && strings.Contains(condition.Status, "True") { + return true, nil + } + } + + return false, nil + }) + + if err != nil { + if wait.Interrupted(err) { + // Try to return a more meaningful error when timed out. + for _, cond := range vnetIf.Status.Conditions { + if strings.Contains(cond.Type, "Ready") && !strings.Contains(cond.Status, "True") { + return nil, fmt.Errorf("network interface is not ready: %s - %s", cond.Reason, cond.Message) + } + } + // TODO: NCP also has an annotation but that usually doesn't provide very useful details. + return nil, fmt.Errorf("network interface is not ready yet") + } + + return nil, err + } + + if vnetIf.Status.ProviderStatus == nil { + return nil, fmt.Errorf("network interface is ready but does not have provider status") + } + + return vnetIf, nil +} + +// ipCIDRNotation takes the IP and subnet mask and returns the IP in CIDR notation. +// TODO: Better error checking. Nail down exactly how we want handle IPv4inV6 addresses. +func ipCIDRNotation(ip string, mask string, isIPv4 bool) string { + if isIPv4 { + ipNet := net.IPNet{ + IP: net.ParseIP(ip).To4(), + Mask: net.IPMask(net.ParseIP(mask).To4()), + } + return ipNet.String() + } + + ipNet := net.IPNet{ + IP: net.ParseIP(ip).To16(), + Mask: net.IPMask(net.ParseIP(mask).To16()), + } + + return ipNet.String() +} + +// CreateDefaultEthCard creates a default Ethernet card attached to the backing. This is used +// when the VM Class ConfigSpec does not have a device entry for a VM Spec network interface, +// so we need a new device. +func CreateDefaultEthCard( + ctx goctx.Context, + result *NetworkInterfaceResult) (vimtypes.BaseVirtualDevice, error) { + + // We may not have the backing yet if this is NSX-T. The backing will be resolved after placement + // when we'll know the CCR, so we can resolve the correct DVPG. + if result.Backing == nil { + return nil, nil + } + + backing, err := result.Backing.EthernetCardBackingInfo(ctx) + if err != nil { + return nil, fmt.Errorf("unable to get ethernet card backing info for network %v: %w", result.Backing.Reference(), err) + } + + dev, err := object.EthernetCardTypes().CreateEthernetCard(defaultEthernetCardType, backing) + if err != nil { + return nil, fmt.Errorf("unable to create ethernet card network %v: %w", result.Backing.Reference(), err) + } + + ethCard := dev.(vimtypes.BaseVirtualEthernetCard).GetVirtualEthernetCard() + ethCard.ExternalId = result.ExternalID + if result.MacAddress != "" { + ethCard.MacAddress = result.MacAddress + ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeManual) + } else { + ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeGenerated) // TODO: Or TypeAssigned? + } + + return dev, nil +} + +// ApplyInterfaceResultToVirtualEthCard applies the interface result from the NetOP/NCP +// provider to an existing Ethernet device from the class ConfigSpec. +func ApplyInterfaceResultToVirtualEthCard( + ctx goctx.Context, + ethCard *vimtypes.VirtualEthernetCard, + result *NetworkInterfaceResult) error { + + ethCard.ExternalId = result.ExternalID + if result.MacAddress != "" { + // BMV: Too much confusion and possible breakage if we don't honor the provider MAC. + // Otherwise, IMO a foot gun and will break on setups that enforce MAC filtering. + ethCard.MacAddress = result.MacAddress + ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeManual) + } else { //nolint + // BMV: IMO this must be Generated/TypeAssigned to avoid major foot gun, but we have tests assuming + // this is left as-is. + // We should have a MAC address field to the VM.Spec if we want this to be specified by the user. + // ethCard.MacAddress = "" + // ethCard.AddressType = string(vimtypes.VirtualEthernetCardMacTypeGenerated) + } + + // We may not have the backing yet if this is NSX-T. The backing will be resolved after placement + // when we'll know the CCR, so we can resolve the correct DVPG. + if result.Backing != nil { + backing, err := result.Backing.EthernetCardBackingInfo(ctx) + if err != nil { + return fmt.Errorf("unable to get ethernet card backing info for network %v: %w", result.NetworkID, err) + } + ethCard.Backing = backing + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/network/network_suite_test.go b/pkg/vmprovider/providers/vsphere2/network/network_suite_test.go new file mode 100644 index 000000000..29fcd237c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/network_suite_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() +var _ = BeforeSuite(suite.BeforeSuite) +var _ = AfterSuite(suite.AfterSuite) + +func TestNetwork(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Network Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/network/network_test.go b/pkg/vmprovider/providers/vsphere2/network/network_test.go new file mode 100644 index 000000000..c57a5e1ab --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/network_test.go @@ -0,0 +1,344 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + goctx "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("CreateAndWaitForNetworkInterfaces", func() { + + var ( + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + interfaceSpecs []vmopv1.VirtualMachineNetworkInterfaceSpec + + results network.NetworkInterfaceResults + err error + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "network-test-vm", + Namespace: "network-test-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithName("network_test"), + VM: vm, + } + + interfaceSpecs = nil + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Context("Named Network", func() { + // Use network vcsim automatically creates. + const networkName = "DC0_DVPG0" + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + }) + + Context("network exists", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: networkName}, + DHCP6: true, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(results.Results).To(HaveLen(1)) + + result := results.Results[0] + By("has expected backing", func() { + Expect(result.Backing).ToNot(BeNil()) + backing, err := result.Backing.EthernetCardBackingInfo(ctx) + Expect(err).ToNot(HaveOccurred()) + backingInfo, ok := backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).To(BeTrue()) + Expect(backingInfo.Port.PortgroupKey).To(Equal(ctx.NetworkRef.Reference().Value)) + }) + + Expect(result.DHCP4).To(BeTrue()) + Expect(result.DHCP6).To(BeTrue()) // Only enabled if explicitly requested (which it is above). + }) + }) + + Context("network does not exist", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: "bogus"}, + }, + } + }) + + It("returns error", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unable to find named network")) + Expect(results.Results).To(BeEmpty()) + }) + }) + }) + + Context("VDS", func() { + const ( + interfaceName = "eth0" + networkName = "my-vds-network" + ) + + BeforeEach(func() { + network.RetryTimeout = 1 * time.Second + testConfig.WithNetworkEnv = builder.NetworkEnvVDS + }) + + Context("Simulate workflow", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: common.PartialObjectRef{ + Name: networkName, + }, + }, + } + }) + + It("returns success", func() { + // Assert test env is what we expect. + Expect(ctx.NetworkRef.Reference().Type).To(Equal("DistributedVirtualPortgroup")) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate successful NetOP reconcile", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.NetworkName).To(Equal(networkName)) + + netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value + netInterface.Status.MacAddress = "" // NetOP doesn't set this. + netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ + { + IP: "192.168.1.110", + IPFamily: netopv1alpha1.IPv4Protocol, + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + { + IP: "fd1a:6c85:79fe:7c98:0000:0000:0000:000f", + IPFamily: netopv1alpha1.IPv6Protocol, + Gateway: "fd1a:6c85:79fe:7c98:0000:0000:0000:0001", + SubnetMask: "ffff:ffff:ffff:ff00:0000:0000:0000:0000", + }, + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.MacAddress).To(BeEmpty()) + Expect(result.ExternalID).To(BeEmpty()) + Expect(result.NetworkID).To(Equal(ctx.NetworkRef.Reference().Value)) + Expect(result.Backing).ToNot(BeNil()) + Expect(result.Backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + Expect(result.Name).To(Equal(interfaceName)) + + Expect(result.IPConfigs).To(HaveLen(2)) + ipConfig := result.IPConfigs[0] + Expect(ipConfig.IPCIDR).To(Equal("192.168.1.110/24")) + Expect(ipConfig.IsIPv4).To(BeTrue()) + Expect(ipConfig.Gateway).To(Equal("192.168.1.1")) + ipConfig = result.IPConfigs[1] + Expect(ipConfig.IPCIDR).To(Equal("fd1a:6c85:79fe:7c98::f/56")) + Expect(ipConfig.IsIPv4).To(BeFalse()) + Expect(ipConfig.Gateway).To(Equal("fd1a:6c85:79fe:7c98:0000:0000:0000:0001")) + }) + }) + }) + + Context("NCP", func() { + const ( + interfaceName = "eth0" + interfaceID = "my-interface-id" + networkName = "my-ncp-network" + macAddress = "01-23-45-67-89-AB-CD-EF" + ) + + BeforeEach(func() { + network.RetryTimeout = 1 * time.Second + testConfig.WithNetworkEnv = builder.NetworkEnvNSXT + }) + + Context("Simulate workflow", func() { + BeforeEach(func() { + interfaceSpecs = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: common.PartialObjectRef{ + Name: networkName, + }, + }, + } + }) + + It("returns success", func() { + // Assert test env is what we expect. + Expect(ctx.NetworkRef.Reference().Type).To(Equal("DistributedVirtualPortgroup")) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate successful NCP reconcile", func() { + netInterface := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NCPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.VirtualNetwork).To(Equal(networkName)) + + netInterface.Status.InterfaceID = interfaceID + netInterface.Status.MacAddress = macAddress + netInterface.Status.ProviderStatus = &ncpv1alpha1.VirtualNetworkInterfaceProviderStatus{ + NsxLogicalSwitchID: builder.NsxTLogicalSwitchUUID, + } + netInterface.Status.IPAddresses = []ncpv1alpha1.VirtualNetworkInterfaceIP{ + { + IP: "192.168.1.110", + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + { + IP: "fd1a:6c85:79fe:7c98:0000:0000:0000:000f", + Gateway: "fd1a:6c85:79fe:7c98:0000:0000:0000:0001", + SubnetMask: "ffff:ffff:ffff:ff00:0000:0000:0000:0000", + }, + } + netInterface.Status.Conditions = []ncpv1alpha1.VirtualNetworkCondition{ + { + Type: "Ready", + Status: "True", + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.MacAddress).To(Equal(macAddress)) + Expect(result.ExternalID).To(Equal(interfaceID)) + Expect(result.NetworkID).To(Equal(builder.NsxTLogicalSwitchUUID)) + Expect(result.Name).To(Equal(interfaceName)) + + Expect(result.IPConfigs).To(HaveLen(2)) + ipConfig := result.IPConfigs[0] + Expect(ipConfig.IPCIDR).To(Equal("192.168.1.110/24")) + Expect(ipConfig.IsIPv4).To(BeTrue()) + Expect(ipConfig.Gateway).To(Equal("192.168.1.1")) + ipConfig = result.IPConfigs[1] + Expect(ipConfig.IPCIDR).To(Equal("fd1a:6c85:79fe:7c98::f/56")) + Expect(ipConfig.IsIPv4).To(BeFalse()) + Expect(ipConfig.Gateway).To(Equal("fd1a:6c85:79fe:7c98:0000:0000:0000:0001")) + + // Without the ClusterMoRef on the first call this will be nil for NSXT. + Expect(result.Backing).To(BeNil()) + + clusterMoRef := ctx.GetSingleClusterCompute().Reference() + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + &clusterMoRef, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + Expect(results.Results).To(HaveLen(1)) + Expect(results.Results[0].Backing).ToNot(BeNil()) + Expect(results.Results[0].Backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/network/nsxt.go b/pkg/vmprovider/providers/vsphere2/network/nsxt.go new file mode 100644 index 000000000..d08c6a6b1 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/nsxt.go @@ -0,0 +1,117 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network + +import ( + goctx "context" + "fmt" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + vimtypes "github.com/vmware/govmomi/vim25/types" +) + +// ResolveBackingPostPlacement fixes up the backings where we did not know the CCR until after +// placement. This should be called if CreateAndWaitForNetworkInterfaces() was called with a nil +// clusterMoRef. Returns true if a backing was resolved, so the ConfigSpec needs to be updated. +func ResolveBackingPostPlacement( + ctx goctx.Context, + vimClient *vim25.Client, + clusterMoRef vimtypes.ManagedObjectReference, + results *NetworkInterfaceResults) (bool, error) { + + if len(results.Results) == 0 { + return false, nil + } + + networkType := lib.GetNetworkProviderType() + if networkType == "" { + return false, fmt.Errorf("no network provider set") + } + + ccr := object.NewClusterComputeResource(vimClient, clusterMoRef) + fixedUp := false + + for idx := range results.Results { + if results.Results[idx].Backing != nil { + continue + } + + var backing object.NetworkReference + var err error + + switch networkType { + case lib.NetworkProviderTypeNSXT: + backing, err = searchNsxtNetworkReference(ctx, ccr, results.Results[idx].NetworkID) + if err != nil { + err = fmt.Errorf("post placement NSX-T backing fixup failed: %w", err) + } + default: + err = fmt.Errorf("only NSX-T networks are expected to need post placement backing fixup") + } + + if err != nil { + return false, err + } + + fixedUp = true + results.Results[idx].Backing = backing + } + + return fixedUp, nil +} + +// searchNsxtNetworkReference takes in NSX-T LogicalSwitchUUID and returns the reference of the network. +func searchNsxtNetworkReference( + ctx goctx.Context, + ccr *object.ClusterComputeResource, + networkID string) (object.NetworkReference, error) { + + // This is more or less how the old code did it. We could save repeated work by moving this + // into the callers since it will always be for the same CCR, but the common case is one NIC, + // or at most a handful, so that's for later. + var obj mo.ClusterComputeResource + if err := ccr.Properties(ctx, ccr.Reference(), []string{"network"}, &obj); err != nil { + return nil, err + } + + var dvpgsMoRefs []vimtypes.ManagedObjectReference + for _, n := range obj.Network { + if n.Type == "DistributedVirtualPortgroup" { + dvpgsMoRefs = append(dvpgsMoRefs, n.Reference()) + } + } + + if len(dvpgsMoRefs) == 0 { + return nil, fmt.Errorf("ClusterComputeResource %s has no DVPGs", ccr.Reference().Value) + } + + var dvpgs []mo.DistributedVirtualPortgroup + err := property.DefaultCollector(ccr.Client()).Retrieve(ctx, dvpgsMoRefs, []string{"config.logicalSwitchUuid"}, &dvpgs) + if err != nil { + return nil, err + } + + var dvpgMoRefs []vimtypes.ManagedObjectReference + for _, dvpg := range dvpgs { + if dvpg.Config.LogicalSwitchUuid == networkID { + dvpgMoRefs = append(dvpgMoRefs, dvpg.Reference()) + } + } + + switch len(dvpgMoRefs) { + case 1: + return object.NewDistributedVirtualPortgroup(ccr.Client(), dvpgMoRefs[0]), nil + case 0: + return nil, fmt.Errorf("no DVPG with NSX-T network ID %q found", networkID) + default: + // The LogicalSwitchUuid is supposed to be unique per CCR, so this is likely an NCP + // misconfiguration, and we don't know which one to pick. + return nil, fmt.Errorf("multiple DVPGs (%d) with NSX-T network ID %q found", len(dvpgMoRefs), networkID) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/network/nsxt_test.go b/pkg/vmprovider/providers/vsphere2/network/nsxt_test.go new file mode 100644 index 000000000..35925553a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/network/nsxt_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package network_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("ResolveBackingPostPlacement", func() { + + var ( + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + + results *network.NetworkInterfaceResults + fixedUp bool + err error + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + + fixedUp, err = network.ResolveBackingPostPlacement( + ctx, + ctx.VCClient.Client, + ctx.GetSingleClusterCompute().Reference(), + results) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Context("returns success", func() { + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNSXT + + results = &network.NetworkInterfaceResults{ + Results: []network.NetworkInterfaceResult{ + { + NetworkID: builder.NsxTLogicalSwitchUUID, + Backing: nil, + }, + }, + } + }) + + It("returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(fixedUp).To(BeTrue()) + + Expect(results.Results).To(HaveLen(1)) + By("should populate the backing", func() { + backing := results.Results[0].Backing + Expect(backing).ToNot(BeNil()) + Expect(backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go new file mode 100644 index 000000000..6a28e8a12 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement.go @@ -0,0 +1,206 @@ +// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + goctx "context" + "fmt" + "strings" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +// Recommendation is the info about a placement recommendation. +type Recommendation struct { + PoolMoRef types.ManagedObjectReference + HostMoRef *types.ManagedObjectReference + // TODO: Datastore, whatever else as we need it. +} + +func relocateSpecToRecommendation(relocateSpec *types.VirtualMachineRelocateSpec) *Recommendation { + // Instance Storage requires the host. + if relocateSpec == nil || relocateSpec.Pool == nil || relocateSpec.Host == nil { + return nil + } + + return &Recommendation{ + PoolMoRef: *relocateSpec.Pool, + HostMoRef: relocateSpec.Host, + } +} + +func clusterPlacementActionToRecommendation(action types.ClusterClusterInitialPlacementAction) *Recommendation { + return &Recommendation{ + PoolMoRef: action.Pool, + HostMoRef: action.TargetHost, + } +} + +func CheckPlacementRelocateSpec(spec *types.VirtualMachineRelocateSpec) error { + if spec == nil { + return fmt.Errorf("RelocateSpec is nil") + } + if spec.Host == nil { + return fmt.Errorf("RelocateSpec does not have a host") + } + if spec.Pool == nil { + return fmt.Errorf("RelocateSpec does not have a resource pool") + } + if spec.Datastore == nil { + return fmt.Errorf("RelocateSpec does not have a datastore") + } + return nil +} + +func ParseRelocateVMResponse( + vmCtx context.VirtualMachineContextA2, + res *types.PlacementResult) *types.VirtualMachineRelocateSpec { + + for _, r := range res.Recommendations { + if r.Reason == string(types.RecommendationReasonCodeXvmotionPlacement) { + for _, a := range r.Action { + if pa, ok := a.(*types.PlacementAction); ok { + if err := CheckPlacementRelocateSpec(pa.RelocateSpec); err != nil { + vmCtx.Logger.V(6).Info("Skipped RelocateSpec", + "reason", err.Error(), "relocateSpec", pa.RelocateSpec) + continue + } + + return pa.RelocateSpec + } + } + } + } + + return nil +} + +func CloneVMRelocateSpec( + vmCtx context.VirtualMachineContextA2, + cluster *object.ClusterComputeResource, + vmRef types.ManagedObjectReference, + cloneSpec *types.VirtualMachineCloneSpec) (*types.VirtualMachineRelocateSpec, error) { + + placementSpec := types.PlacementSpec{ + PlacementType: string(types.PlacementSpecPlacementTypeClone), + CloneSpec: cloneSpec, + RelocateSpec: &cloneSpec.Location, + CloneName: cloneSpec.Config.Name, + Vm: &vmRef, + } + + resp, err := cluster.PlaceVm(vmCtx, placementSpec) + if err != nil { + return nil, err + } + + rSpec := ParseRelocateVMResponse(vmCtx, resp) + if rSpec == nil { + return nil, fmt.Errorf("no valid placement action") + } + + return rSpec, nil +} + +// PlaceVMForCreate determines the suitable placement candidates in the cluster. +func PlaceVMForCreate( + ctx goctx.Context, + cluster *object.ClusterComputeResource, + configSpec *types.VirtualMachineConfigSpec) ([]Recommendation, error) { + + placementSpec := types.PlacementSpec{ + PlacementType: string(types.PlacementSpecPlacementTypeCreate), + ConfigSpec: configSpec, + } + + resp, err := cluster.PlaceVm(ctx, placementSpec) + if err != nil { + return nil, err + } + + var recommendations []Recommendation + + for _, r := range resp.Recommendations { + if r.Reason != string(types.RecommendationReasonCodeXvmotionPlacement) { + continue + } + + for _, a := range r.Action { + if pa, ok := a.(*types.PlacementAction); ok { + if r := relocateSpecToRecommendation(pa.RelocateSpec); r != nil { + recommendations = append(recommendations, *r) + } + } + } + } + + return recommendations, nil +} + +// ClusterPlaceVMForCreate determines the suitable cluster placement among the specified ResourcePools. +func ClusterPlaceVMForCreate( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + resourcePoolsMoRefs []types.ManagedObjectReference, + configSpec *types.VirtualMachineConfigSpec, + needsHost bool) ([]Recommendation, error) { + + // Work around PlaceVmsXCluster bug that crashes vpxd when ConfigSpec.Files is nil. + cs := *configSpec + cs.Files = new(types.VirtualMachineFileInfo) + + placementSpec := types.PlaceVmsXClusterSpec{ + ResourcePools: resourcePoolsMoRefs, + VmPlacementSpecs: []types.PlaceVmsXClusterSpecVmPlacementSpec{ + { + ConfigSpec: cs, + }, + }, + HostRecommRequired: &needsHost, + } + + vmCtx.Logger.V(6).Info("PlaceVmxCluster request", "placementSpec", placementSpec) + + resp, err := object.NewRootFolder(vcClient).PlaceVmsXCluster(vmCtx, placementSpec) + if err != nil { + return nil, err + } + + vmCtx.Logger.V(6).Info("PlaceVmxCluster response", "resp", resp) + + if len(resp.Faults) != 0 { + var faultMgs []string + for _, f := range resp.Faults { + msgs := make([]string, 0, len(f.Faults)) + for _, ff := range f.Faults { + msgs = append(msgs, ff.LocalizedMessage) + } + faultMgs = append(faultMgs, + fmt.Sprintf("ResourcePool %s faults: %s", f.ResourcePool.Value, strings.Join(msgs, ", "))) + } + return nil, fmt.Errorf("PlaceVmsXCluster faults: %v", faultMgs) + } + + var recommendations []Recommendation + + for _, info := range resp.PlacementInfos { + if info.Recommendation.Reason != string(types.RecommendationReasonCodeXClusterPlacement) { + continue + } + + for _, a := range info.Recommendation.Action { + if ca, ok := a.(*types.ClusterClusterInitialPlacementAction); ok { + if r := clusterPlacementActionToRecommendation(*ca); r != nil { + recommendations = append(recommendations, *r) + } + } + } + } + + return recommendations, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go new file mode 100644 index 000000000..8dc78a2a8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/cluster_placement_test.go @@ -0,0 +1,136 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/placement" +) + +func createRelocateSpec() *types.VirtualMachineRelocateSpec { + spec := &types.VirtualMachineRelocateSpec{} + spec.Host = &types.ManagedObjectReference{} + spec.Pool = &types.ManagedObjectReference{} + spec.Datastore = &types.ManagedObjectReference{} + return spec +} + +func createValidPlacementAction() (types.BaseClusterAction, *types.VirtualMachineRelocateSpec) { + action := types.PlacementAction{} + action.RelocateSpec = createRelocateSpec() + return types.BaseClusterAction(&action), action.RelocateSpec +} + +func createInvalidPlacementAction() types.BaseClusterAction { + action := types.PlacementAction{} + action.RelocateSpec = createRelocateSpec() + action.RelocateSpec.Host = nil + return types.BaseClusterAction(&action) +} + +func createStoragePlacementAction() types.BaseClusterAction { + action := types.StoragePlacementAction{} + action.RelocateSpec = *createRelocateSpec() + return types.BaseClusterAction(&action) +} + +func createInvalidRecommendation() types.ClusterRecommendation { + r := types.ClusterRecommendation{} + r.Reason = string(types.RecommendationReasonCodeXvmotionPlacement) + r.Action = append(r.Action, createStoragePlacementAction()) + r.Action = append(r.Action, createInvalidPlacementAction()) + return r +} + +func createValidRecommendation() (types.ClusterRecommendation, *types.VirtualMachineRelocateSpec) { + r := createInvalidRecommendation() + a, s := createValidPlacementAction() + r.Action = append(r.Action, a) + return r, s +} + +var _ = Describe("ParsePlaceVMResponse", func() { + + Context("when response is valid", func() { + Specify("PlaceVm Response is valid", func() { + res := types.PlacementResult{} + res.Recommendations = append(res.Recommendations, createInvalidRecommendation(), createInvalidRecommendation()) + rec, _ := createValidRecommendation() + rec.Reason = string(types.RecommendationReasonCodePowerOnVm) + res.Recommendations = append(res.Recommendations, rec) + rec, spec := createValidRecommendation() + res.Recommendations = append(res.Recommendations, rec) + + rSpec := placement.ParseRelocateVMResponse(&res) + Expect(rSpec).NotTo(BeNil()) + Expect(rSpec.Host).To(BeEquivalentTo(spec.Host)) + Expect(rSpec.Pool).To(BeEquivalentTo(spec.Pool)) + Expect(rSpec.Datastore).To(BeEquivalentTo(spec.Datastore)) + }) + }) + + Context("when response is not valid", func() { + Specify("PlaceVm Response without recommendations", func() { + res := types.PlacementResult{} + rSpec := placement.ParseRelocateVMResponse(&res) + Expect(rSpec).To(BeNil()) + }) + }) + + Context("when response is not valid", func() { + Specify("PlaceVm Response with invalid recommendations only", func() { + res := types.PlacementResult{} + res.Recommendations = append(res.Recommendations, createInvalidRecommendation(), createInvalidRecommendation()) + rec, _ := createValidRecommendation() + rec.Reason = string(types.RecommendationReasonCodePowerOnVm) + res.Recommendations = append(res.Recommendations, rec) + + rSpec := placement.ParseRelocateVMResponse(&res) + Expect(rSpec).To(BeNil()) + }) + }) +}) + +var _ = Describe("CheckPlacementRelocateSpec", func() { + + Context("when relocation spec is valid", func() { + Specify("Relocation spec is valid", func() { + spec := createRelocateSpec() + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeTrue()) + }) + }) + + Context("when relocation spec is not valid", func() { + Specify("Relocation spec is nil", func() { + isValid := placement.CheckPlacementRelocateSpec(nil) + Expect(isValid).To(BeFalse()) + }) + + Specify("Host is nil", func() { + spec := createRelocateSpec() + spec.Host = nil + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeFalse()) + }) + + Specify("Pool is nil", func() { + spec := createRelocateSpec() + spec.Pool = nil + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeFalse()) + }) + + Specify("Datastore is nil", func() { + spec := createRelocateSpec() + spec.Datastore = nil + isValid := placement.CheckPlacementRelocateSpec(spec) + Expect(isValid).To(BeFalse()) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go b/pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go new file mode 100644 index 000000000..377f4133c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/placement_suite_test.go @@ -0,0 +1,26 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("Placement", vcSimPlacement) +} + +var suite = builder.NewTestSuite() + +func TestPlacement(t *testing.T) { + suite.Register(t, "VMProvider Placement", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/placement/zone_placement.go b/pkg/vmprovider/providers/vsphere2/placement/zone_placement.go new file mode 100644 index 000000000..09717487e --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/zone_placement.go @@ -0,0 +1,333 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement + +import ( + goctx "context" + "fmt" + "math/rand" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" +) + +type Result struct { + ZonePlacement bool + InstanceStoragePlacement bool + ZoneName string + HostMoRef *types.ManagedObjectReference + PoolMoRef types.ManagedObjectReference + // TODO: Datastore, whatever else as we need it. +} + +func doesVMNeedPlacement(vmCtx context.VirtualMachineContextA2) (res Result, needZonePlacement, needInstanceStoragePlacement bool) { + if lib.IsWcpFaultDomainsFSSEnabled() { + res.ZonePlacement = true + + if zoneName := vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey]; zoneName != "" { + // Zone has already been selected. + res.ZoneName = zoneName + } else { + // VM does not have a zone already assigned so we need to select one. + needZonePlacement = true + } + } + + if lib.IsInstanceStorageFSSEnabled() { + if instancestorage.IsPresent(vmCtx.VM) { + res.InstanceStoragePlacement = true + + if hostMoID := vmCtx.VM.Annotations[constants.InstanceStorageSelectedNodeMOIDAnnotationKey]; hostMoID != "" { + // Host has already been selected. + res.HostMoRef = &types.ManagedObjectReference{Type: "HostSystem", Value: hostMoID} + } else { + // VM has InstanceStorage volumes so we need to select a host. + needInstanceStoragePlacement = true + } + } + } + + return +} + +// lookupChildRPs lookups the child ResourcePool under each parent ResourcePool. A VM with a ResourcePolicy +// may specify a child ResourcePool that the VM will be created under. +func lookupChildRPs( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + rpMoIDs []string, + zoneName, childRPName string) []string { + + childRPMoIDs := make([]string, 0, len(rpMoIDs)) + + for _, rpMoID := range rpMoIDs { + rp := object.NewResourcePool(vcClient, types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + + childRP, err := vcenter.GetChildResourcePool(vmCtx, rp, childRPName) + if err != nil { + vmCtx.Logger.Error(err, "Skipping this resource pool since failed to get child ResourcePool", + "zone", zoneName, "parentRPMoID", rpMoID, "childRPName", childRPName) + continue + } + + childRPMoIDs = append(childRPMoIDs, childRP.Reference().Value) + } + + return childRPMoIDs +} + +// getPlacementCandidates determines the candidate resource pools for VM placement. +func getPlacementCandidates( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client, + vcClient *vim25.Client, + zonePlacement bool, + childRPName string) (map[string][]string, error) { + + var zones []topologyv1.AvailabilityZone + + if zonePlacement { + z, err := topology.GetAvailabilityZones(vmCtx, client) + if err != nil { + return nil, err + } + + zones = z + } else { + // Consider candidates only within the already assigned zone. + // NOTE: GetAvailabilityZone() will return a "default" AZ when the FSS is not enabled. + zone, err := topology.GetAvailabilityZone(vmCtx, client, vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey]) + if err != nil { + return nil, err + } + + zones = append(zones, zone) + } + + candidates := map[string][]string{} + + for _, zone := range zones { + nsInfo, ok := zone.Spec.Namespaces[vmCtx.VM.Namespace] + if !ok { + continue + } + + var rpMoIDs []string + if len(nsInfo.PoolMoIDs) != 0 { + rpMoIDs = nsInfo.PoolMoIDs + } else { + rpMoIDs = []string{nsInfo.PoolMoId} + } + + if childRPName != "" { + childRPMoIDs := lookupChildRPs(vmCtx, vcClient, rpMoIDs, zone.Name, childRPName) + if len(childRPMoIDs) == 0 { + vmCtx.Logger.Info("Zone had no candidates after looking up children ResourcePools", + "zone", zone.Name, "rpMoIDs", rpMoIDs, "childRPName", childRPName) + continue + } + rpMoIDs = childRPMoIDs + } + + candidates[zone.Name] = rpMoIDs + } + + return candidates, nil +} + +func rpMoIDToCluster( + ctx goctx.Context, + vcClient *vim25.Client, + rpMoRef types.ManagedObjectReference) (*object.ClusterComputeResource, error) { + + cluster, err := object.NewResourcePool(vcClient, rpMoRef).Owner(ctx) + if err != nil { + return nil, err + } + + return object.NewClusterComputeResource(vcClient, cluster.Reference()), nil +} + +// getPlacementRecommendations calls DRS PlaceVM to determine clusters suitable for placement. +func getPlacementRecommendations( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + candidates map[string][]string, + configSpec *types.VirtualMachineConfigSpec) map[string][]Recommendation { + + recommendations := map[string][]Recommendation{} + + for zoneName, rpMoIDs := range candidates { + for _, rpMoID := range rpMoIDs { + rpMoRef := types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID} + + cluster, err := rpMoIDToCluster(vmCtx, vcClient, rpMoRef) + if err != nil { + vmCtx.Logger.Error(err, "failed to get CCR from RP", "zone", zoneName, "rpMoID", rpMoID) + continue + } + + recs, err := PlaceVMForCreate(vmCtx, cluster, configSpec) + if err != nil { + vmCtx.Logger.Error(err, "PlaceVM failed", "zone", zoneName, + "clusterMoID", cluster.Reference().Value, "rpMoID", rpMoID) + continue + } + + if len(recs) == 0 { + vmCtx.Logger.Info("No placement recommendations", "zone", zoneName, + "clusterMoID", cluster.Reference().Value, "rpMoID", rpMoID) + continue + } + + // Replace the resource pool returned by PlaceVM - that is the cluster's root RP - with + // our more specific namespace or namespace child RP since this VM needs to be under the + // more specific RP. This makes the recommendations returned here the same as what zonal + // would return. + for idx := range recs { + recs[idx].PoolMoRef = rpMoRef + } + + recommendations[zoneName] = append(recommendations[zoneName], recs...) + } + } + + vmCtx.Logger.V(5).Info("Placement recommendations", "recommendations", recommendations) + + return recommendations +} + +// getZonalPlacementRecommendations calls DRS PlaceVmsXCluster to determine clusters suitable for placement. +func getZonalPlacementRecommendations( + vmCtx context.VirtualMachineContextA2, + vcClient *vim25.Client, + candidates map[string][]string, + configSpec *types.VirtualMachineConfigSpec, + needsHost bool) map[string][]Recommendation { + + rpMOToZone := map[types.ManagedObjectReference]string{} + var candidateRPMoRefs []types.ManagedObjectReference + + for zoneName, rpMoIDs := range candidates { + for _, rpMoID := range rpMoIDs { + rpMoRef := types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID} + candidateRPMoRefs = append(candidateRPMoRefs, rpMoRef) + rpMOToZone[rpMoRef] = zoneName + } + } + + var recs []Recommendation + + if len(candidateRPMoRefs) == 1 { + // If there is only one candidate, we might be able to skip some work. + + if needsHost { + // This is a hack until PlaceVmsXCluster() supports instance storage disks. + vmCtx.Logger.Info("Falling back into non-zonal placement since the only candidate needs host selected", + "rpMoID", candidateRPMoRefs[0].Value) + return getPlacementRecommendations(vmCtx, vcClient, candidates, configSpec) + } + + recs = append(recs, Recommendation{ + PoolMoRef: candidateRPMoRefs[0], + }) + vmCtx.Logger.V(5).Info("Implied placement since there was only one candidate", "rec", recs[0]) + + } else { + var err error + + recs, err = ClusterPlaceVMForCreate(vmCtx, vcClient, candidateRPMoRefs, configSpec, needsHost) + if err != nil { + vmCtx.Logger.Error(err, "PlaceVmsXCluster failed") + return nil + } + } + + recommendations := map[string][]Recommendation{} + for _, rec := range recs { + if rpZoneName, ok := rpMOToZone[rec.PoolMoRef]; ok { + recommendations[rpZoneName] = append(recommendations[rpZoneName], rec) + } else { + vmCtx.Logger.V(4).Info("Received unexpected ResourcePool recommendation", + "poolMoRef", rec.PoolMoRef) + } + } + + vmCtx.Logger.V(5).Info("Placement recommendations", "recommendations", recommendations) + + return recommendations +} + +// MakePlacementDecision selects one of the recommendations for placement. +func MakePlacementDecision(recommendations map[string][]Recommendation) (string, Recommendation) { + // Use an explicit rand.Intn() instead of first entry returned by map iterator. + zoneNames := make([]string, 0, len(recommendations)) + for zoneName := range recommendations { + zoneNames = append(zoneNames, zoneName) + } + zoneName := zoneNames[rand.Intn(len(zoneNames))] //nolint:gosec + + recs := recommendations[zoneName] + return zoneName, recs[rand.Intn(len(recs))] //nolint:gosec +} + +// Placement determines if the VM needs placement, and if so, determines where to place the VM +// and updates the Labels and Annotations with the placement decision. +func Placement( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client, + vcClient *vim25.Client, + configSpec *types.VirtualMachineConfigSpec, + childRPName string) (*Result, error) { + + existingRes, zonePlacement, instanceStoragePlacement := doesVMNeedPlacement(vmCtx) + if !zonePlacement && !instanceStoragePlacement { + return &existingRes, nil + } + + candidates, err := getPlacementCandidates(vmCtx, client, vcClient, zonePlacement, childRPName) + if err != nil { + return nil, err + } + + if len(candidates) == 0 { + return nil, fmt.Errorf("no placement candidates available") + } + + // TBD: May want to get the host for vGPU and other passthru devices too. + needsHost := instanceStoragePlacement + + var recommendations map[string][]Recommendation + if zonePlacement { + recommendations = getZonalPlacementRecommendations(vmCtx, vcClient, candidates, configSpec, needsHost) + } else /* instanceStoragePlacement */ { + recommendations = getPlacementRecommendations(vmCtx, vcClient, candidates, configSpec) + } + if len(recommendations) == 0 { + return nil, fmt.Errorf("no placement recommendations available") + } + + zoneName, rec := MakePlacementDecision(recommendations) + vmCtx.Logger.V(5).Info("Placement decision result", "zone", zoneName, "recommendation", rec) + + result := &Result{ + ZonePlacement: zonePlacement, + InstanceStoragePlacement: instanceStoragePlacement, + ZoneName: zoneName, + PoolMoRef: rec.PoolMoRef, + HostMoRef: rec.HostMoRef, + } + + return result, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go b/pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go new file mode 100644 index 000000000..674b1ae86 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/placement/zone_placement_test.go @@ -0,0 +1,284 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package placement_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/placement" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("MakePlacementDecision", func() { + + Context("only one placement decision is possible", func() { + It("makes expected decision", func() { + recommendations := map[string][]placement.Recommendation{ + "zone1": { + placement.Recommendation{ + PoolMoRef: types.ManagedObjectReference{Type: "a", Value: "abc"}, + HostMoRef: &types.ManagedObjectReference{Type: "b", Value: "xyz"}, + }, + }, + } + + zoneName, rec := placement.MakePlacementDecision(recommendations) + Expect(zoneName).To(Equal("zone1")) + Expect(rec).To(BeElementOf(recommendations[zoneName])) + }) + }) + + Context("multiple placement candidates exist", func() { + It("makes an decision", func() { + zones := map[string][]string{ + "zone1": {"z1-host1", "z1-host2", "z1-host3"}, + "zone2": {"z2-host1", "z2-host2"}, + } + + recommendations := map[string][]placement.Recommendation{} + for zoneName, hosts := range zones { + for _, host := range hosts { + recommendations[zoneName] = append(recommendations[zoneName], + placement.Recommendation{ + PoolMoRef: types.ManagedObjectReference{Type: "a", Value: "abc"}, + HostMoRef: &types.ManagedObjectReference{Type: "b", Value: host}, + }) + } + } + + zoneName, rec := placement.MakePlacementDecision(recommendations) + Expect(zones).To(HaveKey(zoneName)) + Expect(rec).To(BeElementOf(recommendations[zoneName])) + }) + }) +}) + +func vcSimPlacement() { + + var ( + initObjects []client.Object + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + testConfig builder.VCSimTestConfig + + vm *vmopv1.VirtualMachine + vmCtx context.VirtualMachineContext + configSpec *types.VirtualMachineConfigSpec + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + + vm = builder.DummyVirtualMachine() + vm.Name = "placement-test" + + // Other than the name ConfigSpec contents don't matter for vcsim. + configSpec = &types.VirtualMachineConfigSpec{ + Name: vm.Name, + } + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + nsInfo = ctx.CreateWorkloadNamespace() + + vm.Namespace = nsInfo.Namespace + + vmCtx = context.VirtualMachineContext{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + }) + + Context("Zone placement", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + }) + + Context("zone already assigned", func() { + zoneName := "in the zone" + + JustBeforeEach(func() { + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + }) + + It("returns success with same zone", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + Expect(result).ToNot(BeNil()) + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(Equal(zoneName)) + Expect(result.HostMoRef).To(BeNil()) + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + + // Current contract is the caller must look this up based on the pre-assigned zone but + // we might want to change that later. + Expect(result.PoolMoRef.Value).To(BeEmpty()) + }) + }) + + Context("no placement candidates", func() { + JustBeforeEach(func() { + vm.Namespace = "does-not-exist" + }) + + It("returns an error", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).To(MatchError("no placement candidates available")) + Expect(result).To(BeNil()) + }) + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + Expect(result.PoolMoRef.Value).ToNot(BeEmpty()) + Expect(result.HostMoRef).To(BeNil()) + + nsRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(nsRP.Reference().Value)) + }) + + Context("Only one zone exists", func() { + BeforeEach(func() { + testConfig.NumFaultDomains = 1 + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + Expect(result.PoolMoRef.Value).ToNot(BeEmpty()) + Expect(result.HostMoRef).To(BeNil()) + + nsRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(nsRP.Reference().Value)) + }) + }) + + Context("VM is in child RP via ResourcePolicy", func() { + It("returns success", func() { + resourcePolicy, _ := ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + childRPName := resourcePolicy.Spec.ResourcePool.Name + Expect(childRPName).ToNot(BeEmpty()) + vmCtx.VM.Spec.ResourcePolicyName = resourcePolicy.Name + + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, childRPName) + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + + childRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, childRPName) + Expect(childRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(childRP.Reference().Value)) + }) + }) + }) + + Context("Instance Storage Placement", func() { + + BeforeEach(func() { + testConfig.WithInstanceStorage = true + builder.AddDummyInstanceStorageVolume(vm) + }) + + When("host already assigned", func() { + const hostMoID = "foobar-host-42" + + BeforeEach(func() { + vm.Annotations[constants.InstanceStorageSelectedNodeMOIDAnnotationKey] = hostMoID + }) + + It("returns success with same host", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).To(Equal(hostMoID)) + }) + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).ToNot(BeEmpty()) + }) + + When("FaultDomains FSS is enabled", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + testConfig.NumFaultDomains = 1 // Only support for non-HA "HA" + }) + + It("returns success", func() { + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, "") + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).ToNot(BeEmpty()) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).ToNot(BeEmpty()) + + nsRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(nsRP.Reference().Value)) + }) + + Context("VM is in child RP via ResourcePolicy", func() { + It("returns success", func() { + resourcePolicy, _ := ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + childRPName := resourcePolicy.Spec.ResourcePool.Name + Expect(childRPName).ToNot(BeEmpty()) + vmCtx.VM.Spec.ResourcePolicyName = resourcePolicy.Name + + result, err := placement.Placement(vmCtx, ctx.Client, ctx.VCClient.Client, configSpec, childRPName) + Expect(err).ToNot(HaveOccurred()) + + Expect(result.ZonePlacement).To(BeTrue()) + Expect(result.ZoneName).To(BeElementOf(ctx.ZoneNames)) + + Expect(result.InstanceStoragePlacement).To(BeTrue()) + Expect(result.HostMoRef).ToNot(BeNil()) + Expect(result.HostMoRef.Value).ToNot(BeEmpty()) + + childRP := ctx.GetResourcePoolForNamespace(vm.Namespace, result.ZoneName, childRPName) + Expect(childRP).ToNot(BeNil()) + Expect(result.PoolMoRef.Value).To(Equal(childRP.Reference().Value)) + }) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/resources/vm.go b/pkg/vmprovider/providers/vsphere2/resources/vm.go new file mode 100644 index 000000000..ddc90e713 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/resources/vm.go @@ -0,0 +1,230 @@ +// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + vmutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/vm" +) + +type VirtualMachine struct { + Name string + vcVirtualMachine *object.VirtualMachine + logger logr.Logger +} + +var log = logf.Log.WithName("vmresource") + +func NewVMFromObject(objVM *object.VirtualMachine) *VirtualMachine { + return &VirtualMachine{ + Name: objVM.Name(), + vcVirtualMachine: objVM, + logger: log.WithValues("name", objVM.Name()), + } +} + +func (vm *VirtualMachine) VcVM() *object.VirtualMachine { + return vm.vcVirtualMachine +} + +func (vm *VirtualMachine) Create(ctx context.Context, folder *object.Folder, pool *object.ResourcePool, vmSpec *types.VirtualMachineConfigSpec) error { + vm.logger.V(5).Info("Create VM") + + if vm.vcVirtualMachine != nil { + return fmt.Errorf("failed to create VM %q because the VM object is already set", vm.Name) + } + + createTask, err := folder.CreateVM(ctx, *vmSpec, pool, nil) + if err != nil { + return err + } + + result, err := createTask.WaitForResult(ctx, nil) + if err != nil { + return errors.Wrapf(err, "create VM %q task failed", vm.Name) + } + + vm.vcVirtualMachine = object.NewVirtualMachine(folder.Client(), result.Result.(types.ManagedObjectReference)) + return nil +} + +func (vm *VirtualMachine) Clone(ctx context.Context, folder *object.Folder, cloneSpec *types.VirtualMachineCloneSpec) (*types.ManagedObjectReference, error) { + vm.logger.V(5).Info("Clone VM") + + cloneTask, err := vm.vcVirtualMachine.Clone(ctx, folder, cloneSpec.Config.Name, *cloneSpec) + if err != nil { + return nil, err + } + + result, err := cloneTask.WaitForResult(ctx, nil) + if err != nil { + return nil, errors.Wrapf(err, "clone VM task failed") + } + + ref := result.Result.(types.ManagedObjectReference) + return &ref, nil +} + +func (vm *VirtualMachine) Reconfigure(ctx context.Context, configSpec *types.VirtualMachineConfigSpec) error { + vm.logger.V(5).Info("Reconfiguring VM", "configSpec", configSpec) + + reconfigureTask, err := vm.vcVirtualMachine.Reconfigure(ctx, *configSpec) + if err != nil { + return err + } + + _, err = reconfigureTask.WaitForResult(ctx, nil) + if err != nil { + return errors.Wrapf(err, "reconfigure VM task failed") + } + + return nil +} + +func (vm *VirtualMachine) GetProperties(ctx context.Context, properties []string) (*mo.VirtualMachine, error) { + var o mo.VirtualMachine + err := vm.vcVirtualMachine.Properties(ctx, vm.vcVirtualMachine.Reference(), properties, &o) + if err != nil { + vm.logger.Error(err, "Error getting VM properties") + return nil, err + } + + return &o, nil +} + +func (vm *VirtualMachine) ReferenceValue() string { + vm.logger.V(5).Info("Get ReferenceValue") + return vm.vcVirtualMachine.Reference().Value +} + +func (vm *VirtualMachine) MoRef() types.ManagedObjectReference { + vm.logger.V(5).Info("Get MoRef") + return vm.vcVirtualMachine.Reference() +} + +func (vm *VirtualMachine) UniqueID(ctx context.Context) (string, error) { + // Notes from Alkesh Shah regarding MoIDs in VC as of 7.0 + // + // MoRef IDs are unique within the scope of a single VC. Since Clusters are entities in VCs, the MoRef IDs will be unique across clusters + // + // Identity in VC is derived from a sequence. This ID is used in generating the MoId (or MoRef ID) for the entity in VC. Sequence is monotonically + // increasing and so during regular operation there are no dupes + // + // Backup-Restore: We now make sure that our sequence does not go back in time when restoring from a backup + // ( ) So this + // ensures that after restore we get new MoIds which are never used before… (we advance the sequence counter based on time) + // + // Discovery of VMs: We only use moids from the VM store during restore from a backup. In the unlikely event + // that there are two VMs which are presenting the same MoId, we will regenerate a new MoId based on the current + // sequence. Keep in mind, Ideally the unlikely scenario should not occur as we attempt to tamper proof the MoId + // stored in the VM store ( ) + // so two VMs having the same MoId should not happen because they cannot have the same VMX path and we use VMX path + // for ensuring this tamper proof behavior. + // + // Removing from VC and Re-adding the VM to same VC: VM will be given a new MoId (even if the VM is added using + // RegisterVM operation from VC) + // Basically, lifetime of the identity is tied to VC’s knowledge of it’s existence in it’s inventory + return vm.ReferenceValue(), nil +} + +func (vm *VirtualMachine) SetPowerState( + ctx context.Context, + currentPowerState, + desiredPowerState vmopv1.VirtualMachinePowerState, + desiredPowerOpMode vmopv1.VirtualMachinePowerOpMode) error { + + _, err := vmutil.SetAndWaitOnPowerState( + ctx, + vm.VcVM().Client(), + mo.VirtualMachine{ + ManagedEntity: mo.ManagedEntity{ + ExtensibleManagedObject: mo.ExtensibleManagedObject{ + Self: vm.VcVM().Reference(), + }, + }, + Summary: types.VirtualMachineSummary{ + Runtime: types.VirtualMachineRuntimeInfo{ + PowerState: vmutil.ParsePowerState(string(currentPowerState)), + }, + }, + }, + false, + vmutil.ParsePowerState(string(desiredPowerState)), + vmutil.ParsePowerOpMode(string(desiredPowerOpMode))) + + return err +} + +// GetVirtualDevices returns the VMs VirtualDeviceList. +func (vm *VirtualMachine) GetVirtualDevices(ctx context.Context) (object.VirtualDeviceList, error) { + vm.logger.V(5).Info("GetVirtualDevices") + deviceList, err := vm.vcVirtualMachine.Device(ctx) + if err != nil { + vm.logger.Error(err, "Failed to get devices for VM") + return nil, err + } + + return deviceList, err +} + +// GetVirtualDisks returns the list of VMs vmdks. +func (vm *VirtualMachine) GetVirtualDisks(ctx context.Context) (object.VirtualDeviceList, error) { + vm.logger.V(5).Info("GetVirtualDisks") + deviceList, err := vm.vcVirtualMachine.Device(ctx) + if err != nil { + vm.logger.Error(err, "Failed to get devices for VM") + return nil, err + } + + return deviceList.SelectByType((*types.VirtualDisk)(nil)), nil +} + +func (vm *VirtualMachine) GetNetworkDevices(ctx context.Context) (object.VirtualDeviceList, error) { + vm.logger.V(4).Info("GetNetworkDevices") + devices, err := vm.vcVirtualMachine.Device(ctx) + if err != nil { + vm.logger.Error(err, "Failed to get devices for VM") + return nil, err + } + + return devices.SelectByType((*types.VirtualEthernetCard)(nil)), nil +} + +func (vm *VirtualMachine) Customize(ctx context.Context, spec types.CustomizationSpec) error { + vm.logger.V(5).Info("Customize", "spec", spec) + + customizeTask, err := vm.vcVirtualMachine.Customize(ctx, spec) + if err != nil { + vm.logger.Error(err, "Failed to customize VM") + return err + } + + taskInfo, err := customizeTask.WaitForResult(ctx, nil) + if err != nil { + vm.logger.Error(err, "Failed to wait for the result of Customize VM") + return err + } + + if taskErr := taskInfo.Error; taskErr != nil { + // Fetch fault messages for task.Error + fault := taskErr.Fault.GetMethodFault() + if fault != nil { + err = errors.Wrapf(err, "Fault messages: %v", fault.FaultMessage) + } + + return errors.Wrap(err, "Customization task failed") + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session.go b/pkg/vmprovider/providers/vsphere2/session/session.go new file mode 100644 index 000000000..ea0eb9881 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session.go @@ -0,0 +1,41 @@ +// Copyright (c) 2018-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/internal" + res "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/resources" +) + +type Session struct { + Client *client.Client + K8sClient ctrlruntime.Client + Finder *find.Finder + + // Fields only used during Update + Cluster *object.ClusterComputeResource +} + +func (s *Session) invokeFsrVirtualMachine(vmCtx context.VirtualMachineContextA2, resVM *res.VirtualMachine) error { + vmCtx.Logger.Info("Invoking FSR on VM") + + task, err := internal.VirtualMachineFSR(vmCtx, resVM.MoRef(), s.Client.VimClient()) + if err != nil { + vmCtx.Logger.Error(err, "InvokeFSR call failed") + return err + } + + if err = task.Wait(vmCtx); err != nil { + vmCtx.Logger.Error(err, "InvokeFSR task failed") + return err + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_suite_test.go b/pkg/vmprovider/providers/vsphere2/session/session_suite_test.go new file mode 100644 index 000000000..011bbe5b1 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_suite_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() +var _ = BeforeSuite(suite.BeforeSuite) +var _ = AfterSuite(suite.AfterSuite) + +func TestSession(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider Session Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_util.go b/pkg/vmprovider/providers/vsphere2/session/session_util.go new file mode 100644 index 000000000..fb25dbe91 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_util.go @@ -0,0 +1,34 @@ +// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + vimTypes "github.com/vmware/govmomi/vim25/types" +) + +func ExtraConfigToMap(input []vimTypes.BaseOptionValue) (output map[string]string) { + output = make(map[string]string) + for _, opt := range input { + if optValue := opt.GetOptionValue(); optValue != nil { + // Only set string type values + if val, ok := optValue.Value.(string); ok { + output[optValue.Key] = val + } + } + } + return +} + +// MergeExtraConfig adds the key/value to the ExtraConfig if the key is not present, to let to the value be +// changed by the VM. The existing usage of ExtraConfig is hard to fit in the reconciliation model. +func MergeExtraConfig(extraConfig []vimTypes.BaseOptionValue, newMap map[string]string) []vimTypes.BaseOptionValue { + merged := make([]vimTypes.BaseOptionValue, 0) + ecMap := ExtraConfigToMap(extraConfig) + for k, v := range newMap { + if _, exists := ecMap[k]; !exists { + merged = append(merged, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + return merged +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_util_test.go b/pkg/vmprovider/providers/vsphere2/session/session_util_test.go new file mode 100644 index 000000000..76ca2a0fb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_util_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2019-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" +) + +var _ = Describe("Test Session Utils", func() { + + Context("ExtraConfigToMap", func() { + var ( + extraConfig []vimTypes.BaseOptionValue + extraConfigMap map[string]string + ) + BeforeEach(func() { + extraConfig = []vimTypes.BaseOptionValue{} + }) + JustBeforeEach(func() { + extraConfigMap = session.ExtraConfigToMap(extraConfig) + }) + + Context("Empty extraConfig", func() { + It("Return empty map", func() { + Expect(extraConfigMap).To(HaveLen(0)) + }) + }) + + Context("With extraConfig", func() { + BeforeEach(func() { + extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key1", Value: "value1"}) + extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key2", Value: "value2"}) + }) + It("Return valid map", func() { + Expect(extraConfigMap).To(HaveLen(2)) + Expect(extraConfigMap["key1"]).To(Equal("value1")) + Expect(extraConfigMap["key2"]).To(Equal("value2")) + }) + }) + }) + + Context("MergeExtraConfig", func() { + var ( + extraConfig []vimTypes.BaseOptionValue + newMap map[string]string + merged []vimTypes.BaseOptionValue + ) + BeforeEach(func() { + extraConfig = []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{Key: "existingkey1", Value: "existingvalue1"}, + &vimTypes.OptionValue{Key: "existingkey2", Value: "existingvalue2"}, + } + newMap = map[string]string{} + }) + JustBeforeEach(func() { + merged = session.MergeExtraConfig(extraConfig, newMap) + }) + + Context("Empty newMap", func() { + It("Return empty merged", func() { + Expect(merged).To(BeEmpty()) + }) + }) + + Context("NewMap with existing key", func() { + BeforeEach(func() { + newMap["existingkey1"] = "existingkey1" + }) + It("Return empty merged", func() { + Expect(merged).To(BeEmpty()) + }) + }) + + Context("NewMap with new keys", func() { + BeforeEach(func() { + newMap["newkey1"] = "newvalue1" + newMap["newkey2"] = "newvalue2" + }) + It("Return merged map", func() { + Expect(merged).To(HaveLen(2)) + mergedMap := session.ExtraConfigToMap(merged) + Expect(mergedMap["newkey1"]).To(Equal("newvalue1")) + Expect(mergedMap["newkey2"]).To(Equal("newvalue2")) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm.go b/pkg/vmprovider/providers/vsphere2/session/session_vm.go new file mode 100644 index 000000000..a7aaff105 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm.go @@ -0,0 +1,60 @@ +// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "fmt" + + "github.com/vmware/govmomi/object" + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +func updateVirtualDiskDeviceChanges( + vmCtx context.VirtualMachineContextA2, + virtualDisks object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { + + capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity + if capacity.IsZero() { + return nil, nil + } + + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + found := false + for _, vmDevice := range virtualDisks { + vmDisk, ok := vmDevice.(*vimTypes.VirtualDisk) + if !ok { + continue + } + + // Assume the first disk as the boot disk. We can make this smarter by + // looking at the disk path or whatever else later. + // TODO: De-dupe this with resizeBootDiskDeviceChange() in the clone path. + + newCapacityInBytes := capacity.Value() + if newCapacityInBytes < vmDisk.CapacityInBytes { + err := fmt.Errorf("cannot shrink boot disk from %d bytes to %d bytes", + vmDisk.CapacityInBytes, newCapacityInBytes) + return nil, err + } + + if vmDisk.CapacityInBytes < newCapacityInBytes { + vmDisk.CapacityInBytes = newCapacityInBytes + deviceChanges = append(deviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationEdit, + Device: vmDisk, + }) + } + + found = true + break + } + + if !found { + return nil, fmt.Errorf("could not find the boot disk to change capacity") + } + + return deviceChanges, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go new file mode 100644 index 000000000..1154c20fa --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go @@ -0,0 +1,957 @@ +// Copyright (c) 2018-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session + +import ( + "fmt" + "reflect" + "time" + + "k8s.io/utils/pointer" + + "github.com/go-logr/logr" + "github.com/vmware/govmomi/object" + vimTypes "github.com/vmware/govmomi/vim25/types" + apiEquality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/util" + vmutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/vm" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + network2 "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + res "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/resources" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +const ( + FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" +) + +// VMUpdateArgs contains the arguments needed to update a VM on VC. +type VMUpdateArgs struct { + VMClass *vmopv1.VirtualMachineClass + ResourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + MinCPUFreq uint64 + ExtraConfig map[string]string + + BootstrapData vmlifecycle.BootstrapData + + ConfigSpec *vimTypes.VirtualMachineConfigSpec + + NetworkResults network2.NetworkInterfaceResults + + // hack. Remove after VMSVC-1261. + // indicating if this VM image used is VM service v1alpha1 compatible. + VirtualMachineImageV1Alpha1Compatible bool +} + +func ethCardMatch(newBaseEthCard, curBaseEthCard vimTypes.BaseVirtualEthernetCard) bool { + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + if reflect.TypeOf(curBaseEthCard) != reflect.TypeOf(newBaseEthCard) { + return false + } + } + + curEthCard := curBaseEthCard.GetVirtualEthernetCard() + newEthCard := newBaseEthCard.GetVirtualEthernetCard() + if newEthCard.AddressType == string(vimTypes.VirtualEthernetCardMacTypeManual) { + // If the new card has an assigned MAC address, then it should match with + // the current card. Note only NCP sets the MAC address. + if newEthCard.MacAddress != curEthCard.MacAddress { + return false + } + } + + if newEthCard.ExternalId != "" { + // If the new card has a specific ExternalId, then it should match with the + // current card. Note only NCP sets the ExternalId. + if newEthCard.ExternalId != curEthCard.ExternalId { + return false + } + } + + return true +} + +func UpdateEthCardDeviceChanges( + expectedEthCards object.VirtualDeviceList, + currentEthCards object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { + + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + for _, expectedDev := range expectedEthCards { + expectedNic := expectedDev.(vimTypes.BaseVirtualEthernetCard) + expectedBacking := expectedNic.GetVirtualEthernetCard().Backing + expectedBackingType := reflect.TypeOf(expectedBacking) + + var matchingIdx = -1 + + // Try to match the expected NIC with an existing NIC but this isn't that great. We mostly + // depend on the backing but we can improve that later on. When not generated, we could use + // the MAC address. When we support something other than just vmxnet3 we should compare + // those types too. And we should make this truly reconcile as well by comparing the full + // state (support EDIT instead of only ADD/REMOVE operations). + // + // Another tack we could take is force the VM's device order to match the Spec order, but + // that could lead to spurious removals. Or reorder the NetIfList to not be that of the + // Spec, but in VM device order. + for idx, curDev := range currentEthCards { + nic := curDev.(vimTypes.BaseVirtualEthernetCard) + + // This assumes we don't have multiple NICs in the same backing network. This is kind of, sort + // of enforced by the webhook, but we lack a guaranteed way to match up the NICs. + + if !ethCardMatch(expectedNic, nic) { + continue + } + + db := nic.GetVirtualEthernetCard().Backing + if db == nil || reflect.TypeOf(db) != expectedBackingType { + continue + } + + var backingMatch bool + + // Cribbed from VirtualDeviceList.SelectByBackingInfo(). + switch a := db.(type) { + case *vimTypes.VirtualEthernetCardNetworkBackingInfo: + // This backing is only used in testing. + b := expectedBacking.(*vimTypes.VirtualEthernetCardNetworkBackingInfo) + backingMatch = a.DeviceName == b.DeviceName + case *vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo: + b := expectedBacking.(*vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo) + backingMatch = a.Port.SwitchUuid == b.Port.SwitchUuid && a.Port.PortgroupKey == b.Port.PortgroupKey + case *vimTypes.VirtualEthernetCardOpaqueNetworkBackingInfo: + b := expectedBacking.(*vimTypes.VirtualEthernetCardOpaqueNetworkBackingInfo) + backingMatch = a.OpaqueNetworkId == b.OpaqueNetworkId + } + + if backingMatch { + matchingIdx = idx + break + } + } + + if matchingIdx == -1 { + // No matching backing found so add new card. + deviceChanges = append(deviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Device: expectedDev, + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + }) + } else { + // Matching backing found so keep this card (don't remove it below after this loop). + currentEthCards = append(currentEthCards[:matchingIdx], currentEthCards[matchingIdx+1:]...) + } + } + + // Remove any unmatched existing interfaces. + removeDeviceChanges := make([]vimTypes.BaseVirtualDeviceConfigSpec, 0, len(currentEthCards)) + for _, dev := range currentEthCards { + removeDeviceChanges = append(removeDeviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Device: dev, + Operation: vimTypes.VirtualDeviceConfigSpecOperationRemove, + }) + } + + // Process any removes first. + return append(removeDeviceChanges, deviceChanges...), nil +} + +// UpdatePCIDeviceChanges returns devices changes for PCI devices attached to a VM. There are 2 types of PCI devices +// processed here and in case of cloning a VM, devices listed in VMClass are considered as source of truth. +func UpdatePCIDeviceChanges( + expectedPciDevices object.VirtualDeviceList, + currentPciDevices object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { + + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + for _, expectedDev := range expectedPciDevices { + expectedPci := expectedDev.(*vimTypes.VirtualPCIPassthrough) + expectedBacking := expectedPci.Backing + expectedBackingType := reflect.TypeOf(expectedBacking) + + var matchingIdx = -1 + for idx, curDev := range currentPciDevices { + curBacking := curDev.GetVirtualDevice().Backing + if curBacking == nil || reflect.TypeOf(curBacking) != expectedBackingType { + continue + } + + var backingMatch bool + switch a := curBacking.(type) { + case *vimTypes.VirtualPCIPassthroughVmiopBackingInfo: + b := expectedBacking.(*vimTypes.VirtualPCIPassthroughVmiopBackingInfo) + backingMatch = a.Vgpu == b.Vgpu + + case *vimTypes.VirtualPCIPassthroughDynamicBackingInfo: + currAllowedDevs := a.AllowedDevice + b := expectedBacking.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo) + if a.CustomLabel == b.CustomLabel { + // b.AllowedDevice has only one element because CreatePCIDevices() adds only one device based + // on the devices listed in vmclass.spec.hardware.devices.dynamicDirectPathIODevices. + expectedAllowedDev := b.AllowedDevice[0] + for i := 0; i < len(currAllowedDevs) && !backingMatch; i++ { + backingMatch = expectedAllowedDev.DeviceId == currAllowedDevs[i].DeviceId && + expectedAllowedDev.VendorId == currAllowedDevs[i].VendorId + } + } + } + + if backingMatch { + matchingIdx = idx + break + } + } + + if matchingIdx == -1 { + deviceChanges = append(deviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + Device: expectedPci, + }) + } else { + // There could be multiple vGPUs with same BackingInfo. Remove current device if matching found. + currentPciDevices = append(currentPciDevices[:matchingIdx], currentPciDevices[matchingIdx+1:]...) + } + } + // Remove any unmatched existing devices. + removeDeviceChanges := make([]vimTypes.BaseVirtualDeviceConfigSpec, 0, len(currentPciDevices)) + for _, dev := range currentPciDevices { + removeDeviceChanges = append(removeDeviceChanges, &vimTypes.VirtualDeviceConfigSpec{ + Device: dev, + Operation: vimTypes.VirtualDeviceConfigSpecOperationRemove, + }) + } + + // Process any removes first. + return append(removeDeviceChanges, deviceChanges...), nil +} + +func UpdateConfigSpecCPUAllocation( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec, + minCPUFreq uint64) { + + cpuAllocation := config.CpuAllocation + var cpuReservation *int64 + var cpuLimit *int64 + + if !vmClassSpec.Policies.Resources.Requests.Cpu.IsZero() { + rsv := virtualmachine.CPUQuantityToMhz(vmClassSpec.Policies.Resources.Requests.Cpu, minCPUFreq) + if cpuAllocation == nil || cpuAllocation.Reservation == nil || *cpuAllocation.Reservation != rsv { + cpuReservation = &rsv + } + } + + if !vmClassSpec.Policies.Resources.Limits.Cpu.IsZero() { + lim := virtualmachine.CPUQuantityToMhz(vmClassSpec.Policies.Resources.Limits.Cpu, minCPUFreq) + if cpuAllocation == nil || cpuAllocation.Limit == nil || *cpuAllocation.Limit != lim { + cpuLimit = &lim + } + } + + if cpuReservation != nil || cpuLimit != nil { + configSpec.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: cpuReservation, + Limit: cpuLimit, + } + } +} + +func UpdateConfigSpecMemoryAllocation( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec) { + + memAllocation := config.MemoryAllocation + var memoryReservation *int64 + var memoryLimit *int64 + + if !vmClassSpec.Policies.Resources.Requests.Memory.IsZero() { + rsv := virtualmachine.MemoryQuantityToMb(vmClassSpec.Policies.Resources.Requests.Memory) + if memAllocation == nil || memAllocation.Reservation == nil || *memAllocation.Reservation != rsv { + memoryReservation = &rsv + } + } + + if !vmClassSpec.Policies.Resources.Limits.Memory.IsZero() { + lim := virtualmachine.MemoryQuantityToMb(vmClassSpec.Policies.Resources.Limits.Memory) + if memAllocation == nil || memAllocation.Limit == nil || *memAllocation.Limit != lim { + memoryLimit = &lim + } + } + + if memoryReservation != nil || memoryLimit != nil { + configSpec.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: memoryReservation, + Limit: memoryLimit, + } + } +} + +func UpdateConfigSpecExtraConfig( + config *vimTypes.VirtualMachineConfigInfo, + configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec, + vm *vmopv1.VirtualMachine, + globalExtraConfig map[string]string, + imageV1Alpha1Compatible bool) { + + extraConfig := make(map[string]string) + for k, v := range globalExtraConfig { + extraConfig[k] = v + } + + virtualDevices := vmClassSpec.Hardware.Devices + pciPassthruFromConfigSpec := util.SelectVirtualPCIPassthrough(util.DevicesFromConfigSpec(classConfigSpec)) + if len(virtualDevices.VGPUDevices) > 0 || len(virtualDevices.DynamicDirectPathIODevices) > 0 || len(pciPassthruFromConfigSpec) > 0 { + // Add "maintenance.vm.evacuation.poweroff" extraConfig key when GPU devices are present in the VMClass Spec. + extraConfig[constants.MMPowerOffVMExtraConfigKey] = constants.ExtraConfigTrue + setMMIOExtraConfig(vm, extraConfig) + } + + // If VM has InstanceStorage configured, add "maintenance.vm.evacuation.poweroff" to extraConfig + if instancestorage.IsPresent(vm) { + extraConfig[constants.MMPowerOffVMExtraConfigKey] = constants.ExtraConfigTrue + } + + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + // Merge non intersecting keys from the desired config spec extra config with the class config spec extra config + // (ie) class config spec extra config keys takes precedence over the desired config spec extra config keys + ecFromClassConfigSpec := ExtraConfigToMap(classConfigSpec.ExtraConfig) + mergedExtraConfig := classConfigSpec.ExtraConfig + for k, v := range extraConfig { + if _, exists := ecFromClassConfigSpec[k]; !exists { + mergedExtraConfig = append(mergedExtraConfig, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + extraConfig = ExtraConfigToMap(mergedExtraConfig) + } + + configSpec.ExtraConfig = MergeExtraConfig(config.ExtraConfig, extraConfig) + + // Enabling the defer-cloud-init extraConfig key for V1Alpha1Compatible images defers cloud-init from running on first boot + // and disables networking configurations by cloud-init. Therefore, only set the extraConfig key to enabled + // when the vmMetadata is nil or when the transport requested is not CloudInit. + // VMSVC-1261: we may always set this extra config key to remove image from VM customization. + // If a VM is deployed from an incompatible image, + // it will do nothing and won't cause any issues, but can introduce confusion. + // BMV: Is this needed anymore? IMO we shouldn't have bootstrap stuff here. The EC mangling is already hard to follow. + emptyBSSpec := vmopv1.VirtualMachineBootstrapSpec{} + if vm.Spec.Bootstrap == emptyBSSpec || vm.Spec.Bootstrap.CloudInit == nil { + ecMap := ExtraConfigToMap(config.ExtraConfig) + if ecMap[constants.VMOperatorV1Alpha1ExtraConfigKey] == constants.VMOperatorV1Alpha1ConfigReady && + imageV1Alpha1Compatible { + // Set VMOperatorV1Alpha1ExtraConfigKey for v1alpha1 VirtualMachineImage compatibility. + configSpec.ExtraConfig = append(configSpec.ExtraConfig, + &vimTypes.OptionValue{Key: constants.VMOperatorV1Alpha1ExtraConfigKey, Value: constants.VMOperatorV1Alpha1ConfigEnabled}) + } + } +} + +func setMMIOExtraConfig(vm *vmopv1.VirtualMachine, extraConfig map[string]string) { + mmioSize := vm.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] + if mmioSize == "" { + mmioSize = constants.PCIPassthruMMIOSizeDefault + } + if mmioSize != "0" { + extraConfig[constants.PCIPassthruMMIOExtraConfigKey] = constants.ExtraConfigTrue + extraConfig[constants.PCIPassthruMMIOSizeExtraConfigKey] = mmioSize + } +} + +func UpdateConfigSpecChangeBlockTracking( + config *vimTypes.VirtualMachineConfigInfo, + configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec, + vmSpec vmopv1.VirtualMachineSpec) { + + // When VM_Class_as_Config_DaynDate is enabled, class config spec cbt if + // set overrides the VM spec advanced options cbt. + // BMV: I don't think this is correct: the class shouldn't dictate this for backup purposes. There is a + // webhook out there that changes this in the VM spec. + if lib.IsVMClassAsConfigFSSDaynDateEnabled() && classConfigSpec != nil { + if classConfigSpec.ChangeTrackingEnabled != nil { + if !apiEquality.Semantic.DeepEqual(config.ChangeTrackingEnabled, classConfigSpec.ChangeTrackingEnabled) { + configSpec.ChangeTrackingEnabled = classConfigSpec.ChangeTrackingEnabled + } + return + } + } + + if vmSpec.Advanced.ChangeBlockTracking { + if config.ChangeTrackingEnabled == nil || !*config.ChangeTrackingEnabled { + configSpec.ChangeTrackingEnabled = pointer.Bool(true) + } + } else { + if config.ChangeTrackingEnabled != nil && *config.ChangeTrackingEnabled { + configSpec.ChangeTrackingEnabled = pointer.Bool(false) + } + } +} + +func UpdateHardwareConfigSpec( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec) { + + if nCPUs := int32(vmClassSpec.Hardware.Cpus); config.Hardware.NumCPU != nCPUs { + configSpec.NumCPUs = nCPUs + } + if memMB := virtualmachine.MemoryQuantityToMb(vmClassSpec.Hardware.Memory); int64(config.Hardware.MemoryMB) != memMB { + configSpec.MemoryMB = memMB + } +} + +func UpdateConfigSpecAnnotation( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec) { + if config.Annotation != constants.VCVMAnnotation { + configSpec.Annotation = constants.VCVMAnnotation + } +} + +func UpdateConfigSpecManagedBy( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec) { + if config.ManagedBy == nil { + configSpec.ManagedBy = &vimTypes.ManagedByInfo{ + ExtensionKey: constants.ManagedByExtensionKey, + Type: constants.ManagedByExtensionType, + } + } +} + +func UpdateConfigSpecFirmware( + config *vimTypes.VirtualMachineConfigInfo, + configSpec *vimTypes.VirtualMachineConfigSpec, + vm *vmopv1.VirtualMachine) { + + if val, ok := vm.Annotations[constants.FirmwareOverrideAnnotation]; ok { + if (val == "efi" || val == "bios") && config.Firmware != val { + configSpec.Firmware = val + } + } +} + +// UpdateConfigSpecDeviceGroups sets the desired config spec device groups to reconcile by differencing the +// current VM config and the class config spec device groups. +func UpdateConfigSpecDeviceGroups( + config *vimTypes.VirtualMachineConfigInfo, + configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec) { + + if classConfigSpec.DeviceGroups != nil { + if config.DeviceGroups == nil || !reflect.DeepEqual(classConfigSpec.DeviceGroups.DeviceGroup, config.DeviceGroups.DeviceGroup) { + configSpec.DeviceGroups = classConfigSpec.DeviceGroups + } + } +} + +// updateConfigSpec overlays the VM Class spec with the provided ConfigSpec to form a desired +// ConfigSpec that will be used to reconfigure the VM. +func updateConfigSpec( + vmCtx context.VirtualMachineContextA2, + config *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) *vimTypes.VirtualMachineConfigSpec { + + configSpec := &vimTypes.VirtualMachineConfigSpec{} + vmClassSpec := updateArgs.VMClass.Spec + + // Before VM Class as Config, VMs were deployed from the OVA, and are then + // reconfigured to match the desired CPU and memory reservation. Maintain that + // behavior. With the FSS enabled, VMs will be _created_ with desired HW spec, and we + // will not modify the hardware of the VM post creation. So, don't populate the + // Hardware config and CPU/Memory reservation. + if !lib.IsVMClassAsConfigFSSDaynDateEnabled() { + UpdateHardwareConfigSpec(config, configSpec, &vmClassSpec) + UpdateConfigSpecCPUAllocation(config, configSpec, &vmClassSpec, updateArgs.MinCPUFreq) + UpdateConfigSpecMemoryAllocation(config, configSpec, &vmClassSpec) + } + + UpdateConfigSpecAnnotation(config, configSpec) + UpdateConfigSpecManagedBy(config, configSpec) + UpdateConfigSpecExtraConfig(config, configSpec, updateArgs.ConfigSpec, &vmClassSpec, + vmCtx.VM, updateArgs.ExtraConfig, updateArgs.VirtualMachineImageV1Alpha1Compatible) + UpdateConfigSpecChangeBlockTracking(config, configSpec, updateArgs.ConfigSpec, vmCtx.VM.Spec) + UpdateConfigSpecFirmware(config, configSpec, vmCtx.VM) + UpdateConfigSpecDeviceGroups(config, configSpec, updateArgs.ConfigSpec) + + return configSpec +} + +func (s *Session) prePowerOnVMConfigSpec( + vmCtx context.VirtualMachineContextA2, + config *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) (*vimTypes.VirtualMachineConfigSpec, error) { + + configSpec := updateConfigSpec(vmCtx, config, updateArgs) + + virtualDevices := object.VirtualDeviceList(config.Hardware.Device) + currentDisks := virtualDevices.SelectByType((*vimTypes.VirtualDisk)(nil)) + currentEthCards := virtualDevices.SelectByType((*vimTypes.VirtualEthernetCard)(nil)) + currentPciDevices := virtualDevices.SelectByType((*vimTypes.VirtualPCIPassthrough)(nil)) + + diskDeviceChanges, err := updateVirtualDiskDeviceChanges(vmCtx, currentDisks) + if err != nil { + return nil, err + } + configSpec.DeviceChange = append(configSpec.DeviceChange, diskDeviceChanges...) + + var expectedEthCards object.VirtualDeviceList + for idx := range updateArgs.NetworkResults.Results { + expectedEthCards = append(expectedEthCards, updateArgs.NetworkResults.Results[idx].Device) + } + + ethCardDeviceChanges, err := UpdateEthCardDeviceChanges(expectedEthCards, currentEthCards) + if err != nil { + return nil, err + } + configSpec.DeviceChange = append(configSpec.DeviceChange, ethCardDeviceChanges...) + + var expectedPCIDevices []vimTypes.BaseVirtualDevice + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + if configSpecDevs := util.DevicesFromConfigSpec(updateArgs.ConfigSpec); len(configSpecDevs) > 0 { + pciPassthruFromConfigSpec := util.SelectVirtualPCIPassthrough(configSpecDevs) + expectedPCIDevices = virtualmachine.CreatePCIDevicesFromConfigSpec(pciPassthruFromConfigSpec) + } + } else { + expectedPCIDevices = virtualmachine.CreatePCIDevicesFromVMClass(updateArgs.VMClass.Spec.Hardware.Devices) + } + + pciDeviceChanges, err := UpdatePCIDeviceChanges(expectedPCIDevices, currentPciDevices) + if err != nil { + return nil, err + } + configSpec.DeviceChange = append(configSpec.DeviceChange, pciDeviceChanges...) + + return configSpec, nil +} + +func (s *Session) prePowerOnVMReconfigure( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) error { + + configSpec, err := s.prePowerOnVMConfigSpec(vmCtx, config, updateArgs) + if err != nil { + return err + } + + defaultConfigSpec := &vimTypes.VirtualMachineConfigSpec{} + if !apiEquality.Semantic.DeepEqual(configSpec, defaultConfigSpec) { + vmCtx.Logger.Info("Pre PowerOn Reconfigure", "configSpec", configSpec) + if err := resVM.Reconfigure(vmCtx, configSpec); err != nil { + vmCtx.Logger.Error(err, "pre power on reconfigure failed") + return err + } + } + + return nil +} + +func (s *Session) ensureNetworkInterfaces( + vmCtx context.VirtualMachineContextA2, + configSpec *vimTypes.VirtualMachineConfigSpec) (network2.NetworkInterfaceResults, error) { + + // This negative device key is the traditional range used for network interfaces. + deviceKey := int32(-100) + + var networkDevices []vimTypes.BaseVirtualDevice + if lib.IsVMClassAsConfigFSSDaynDateEnabled() && configSpec != nil { + networkDevices = util.SelectDevicesByTypes( + util.DevicesFromConfigSpec(configSpec), + &vimTypes.VirtualE1000{}, + &vimTypes.VirtualE1000e{}, + &vimTypes.VirtualPCNet32{}, + &vimTypes.VirtualVmxnet2{}, + &vimTypes.VirtualVmxnet3{}, + &vimTypes.VirtualVmxnet3Vrdma{}, + &vimTypes.VirtualSriovEthernetCard{}, + ) + } + networkSpec := &vmCtx.VM.Spec.Network + if networkSpec.Disabled { + // No connected networking for this VM. + return network2.NetworkInterfaceResults{}, nil + } + + interfaces := networkSpec.Interfaces + if len(interfaces) == 0 { + // VM gets one automatic NIC. Create the default interface from fields in the network spec. + defaultInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ + Name: networkSpec.DeviceName, + Addresses: networkSpec.Addresses, + DHCP4: networkSpec.DHCP4, + DHCP6: networkSpec.DHCP6, + Gateway4: networkSpec.Gateway4, + Gateway6: networkSpec.Gateway6, + MTU: networkSpec.MTU, + Nameservers: networkSpec.Nameservers, + Routes: networkSpec.Routes, + SearchDomains: networkSpec.SearchDomains, + } + + if defaultInterface.Name == "" { + defaultInterface.Name = "eth0" + } + if networkSpec.Network != nil { + defaultInterface.Network = *networkSpec.Network + } + + interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{defaultInterface} + } + + clusterMoRef := s.Cluster.Reference() + results, err := network2.CreateAndWaitForNetworkInterfaces( + vmCtx, + s.K8sClient, + s.Client.VimClient(), + s.Finder, + &clusterMoRef, + interfaces) + if err != nil { + return network2.NetworkInterfaceResults{}, err + } + + // XXX: The following logic assumes that the order of network interfaces specified in the + // VM spec matches one to one with the device changes in the ConfigSpec in VM class. + // This is a safe assumption for now since VM service only supports one network interface. + // TODO: Needs update when VM Service supports VMs with more then one network interface. + for idx := range results.Results { + result := &results.Results[idx] + + dev, err := network2.CreateDefaultEthCard(vmCtx, result) + if err != nil { + return network2.NetworkInterfaceResults{}, err + } + + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + // If VM Class-as-a-Config is supported, we use the network device from the Class. + // If the VM class doesn't specify enough number of network devices, we fall back to default behavior. + if idx < len(networkDevices) { + ethCardFromNetProvider := dev.(vimTypes.BaseVirtualEthernetCard) + + if mac := ethCardFromNetProvider.GetVirtualEthernetCard().MacAddress; mac != "" { + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().MacAddress = mac + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().AddressType = string(vimTypes.VirtualEthernetCardMacTypeManual) + } + + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().ExternalId = + ethCardFromNetProvider.GetVirtualEthernetCard().ExternalId + // If the device from VM class has a DVX backing, this should still work if the backing as well + // as the DVX backing are set. VPXD checks for DVX backing before checking for normal device backings. + networkDevices[idx].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().Backing = + ethCardFromNetProvider.GetVirtualEthernetCard().Backing + + dev = networkDevices[idx] + } + } + + // govmomi assigns a random device key. Fix that up here. + dev.GetVirtualDevice().Key = deviceKey + deviceKey-- + + result.Device = dev + } + + return results, nil +} + +func (s *Session) ensureCNSVolumes(vmCtx context.VirtualMachineContextA2) error { + // If VM spec has a PVC, check if the volume is attached before powering on + for _, volume := range vmCtx.VM.Spec.Volumes { + if volume.PersistentVolumeClaim == nil { + // Only handle PVC volumes here. In v1a1 we had non-PVC ("vsphereVolumes") but those are gone. + continue + } + + // BMV: We should not use the Status as the SoT here. What a mess. + found := false + for _, volumeStatus := range vmCtx.VM.Status.Volumes { + if volumeStatus.Name == volume.Name { + found = true + if !volumeStatus.Attached { + return fmt.Errorf("persistent volume: %s not attached to VM", volume.Name) + } + break + } + } + + if !found { + return fmt.Errorf("status update pending for persistent volume: %s on VM", volume.Name) + } + } + + return nil +} + +func (s *Session) customize( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + cfg *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) error { + + { + // Hack: the old code only populated the first nic address - getFirstNicMacAddr() - so just keep + // doing the same here. We need a generalized UpdateEthCardDeviceChanges() to match up the Spec + // with the actual devices. Old code also made this best effort so do that here too. + // I've got a larger change that removes the old session stuff, and improves on all this behavior + // but I didn't have the BW to sort out all the changes. + if len(updateArgs.NetworkResults.Results) > 1 { + mac := updateArgs.NetworkResults.Results[0].MacAddress + if mac == "" { + ethCards, _ := resVM.GetNetworkDevices(vmCtx) + if len(ethCards) > 0 { + curNic := ethCards[0].(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard() + updateArgs.NetworkResults.Results[0].MacAddress = curNic.GetVirtualEthernetCard().MacAddress + } + } + } + } + + err := vmlifecycle.DoBootstrap(vmCtx, resVM.VcVM(), cfg, s.K8sClient, updateArgs.NetworkResults, updateArgs.BootstrapData) + if err != nil { + return err + } + + return nil +} + +func (s *Session) prepareVMForPowerOn( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + cfg *vimTypes.VirtualMachineConfigInfo, + updateArgs *VMUpdateArgs) error { + + netIfList, err := s.ensureNetworkInterfaces(vmCtx, updateArgs.ConfigSpec) + if err != nil { + return err + } + + updateArgs.NetworkResults = netIfList + + err = s.prePowerOnVMReconfigure(vmCtx, resVM, cfg, updateArgs) + if err != nil { + return err + } + + err = s.customize(vmCtx, resVM, cfg, updateArgs) + if err != nil { + return err + } + + err = s.ensureCNSVolumes(vmCtx) + if err != nil { + return err + } + + return nil +} + +func (s *Session) poweredOnVMReconfigure( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo) error { + + configSpec := &vimTypes.VirtualMachineConfigSpec{} + UpdateConfigSpecChangeBlockTracking(config, configSpec, nil, vmCtx.VM.Spec) + + defaultConfigSpec := &vimTypes.VirtualMachineConfigSpec{} + if !apiEquality.Semantic.DeepEqual(configSpec, defaultConfigSpec) { + vmCtx.Logger.Info("PoweredOn Reconfigure", "configSpec", configSpec) + if err := resVM.Reconfigure(vmCtx, configSpec); err != nil { + vmCtx.Logger.Error(err, "powered on reconfigure failed") + return err + } + + // Special case for CBT: in order for CBT change take effect for a powered on VM, + // a checkpoint save/restore is needed. tracks the implementation of + // this FSR internally to vSphere. + if configSpec.ChangeTrackingEnabled != nil { + if err := s.invokeFsrVirtualMachine(vmCtx, resVM); err != nil { + vmCtx.Logger.Error(err, "Failed to invoke FSR for CBT update") + return err + } + } + } + + return nil +} + +func (s *Session) attachClusterModule( + vmCtx context.VirtualMachineContextA2, + resVM *res.VirtualMachine, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + // The clusterModule is required be able to enforce the vm-vm anti-affinity policy. + clusterModuleName := vmCtx.VM.Annotations[pkg.ClusterModuleNameKey] + if clusterModuleName == "" { + return nil + } + + // Find ClusterModule UUID from the ResourcePolicy. + _, moduleUUID := clustermodules.FindClusterModuleUUID(clusterModuleName, s.Cluster.Reference(), resourcePolicy) + if moduleUUID == "" { + return fmt.Errorf("ClusterModule %s not found", clusterModuleName) + } + + return s.Client.ClusterModuleClient().AddMoRefToModule(vmCtx, moduleUUID, resVM.MoRef()) +} + +func (s *Session) UpdateVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + getUpdateArgsFn func() (*VMUpdateArgs, error)) (err error) { + + resVM := res.NewVMFromObject(vcVM) + + moVM, err := resVM.GetProperties(vmCtx, []string{"config", "runtime"}) + if err != nil { + return err + } + + defer func() { + updateErr := vmlifecycle.UpdateStatus(vmCtx, s.K8sClient, vcVM, nil) + if updateErr != nil { + vmCtx.Logger.Error(updateErr, "Updating VM status failed") + if err == nil { + err = updateErr + } + } + }() + + // Translate the VM's current power state into the VM Op power state value. + var existingPowerState vmopv1.VirtualMachinePowerState + switch moVM.Runtime.PowerState { + case vimTypes.VirtualMachinePowerStatePoweredOn: + existingPowerState = vmopv1.VirtualMachinePowerStateOn + case vimTypes.VirtualMachinePowerStatePoweredOff: + existingPowerState = vmopv1.VirtualMachinePowerStateOff + case vimTypes.VirtualMachinePowerStateSuspended: + existingPowerState = vmopv1.VirtualMachinePowerStateSuspended + } + + switch vmCtx.VM.Spec.PowerState { + case vmopv1.VirtualMachinePowerStateOff: + var powerOff bool + if existingPowerState == vmopv1.VirtualMachinePowerStateOn { + powerOff = true + } else if existingPowerState == vmopv1.VirtualMachinePowerStateSuspended { + if vmCtx.VM.Spec.PowerOffMode == vmopv1.VirtualMachinePowerOpModeHard { + powerOff = true + } + } + if powerOff { + return resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmCtx.VM.Spec.PowerOffMode) + } + + // BMV: We'll likely want to reconfigure a powered off VM too, but right now + // we'll defer that until the pre power on (and until more people complain + // that the UI appears wrong). + + case vmopv1.VirtualMachinePowerStateSuspended: + if existingPowerState == vmopv1.VirtualMachinePowerStateOn { + return resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmCtx.VM.Spec.SuspendMode) + } + + case vmopv1.VirtualMachinePowerStateOn: + config := moVM.Config + + // See GoVmomi's VirtualMachine::Device() explanation for this check. + if config == nil { + return fmt.Errorf("VM config is not available, connectionState=%s", moVM.Runtime.ConnectionState) + } + + switch existingPowerState { + case vmopv1.VirtualMachinePowerStateOn: + + // Check to see if a possible restart is required. + // Please note a VM may only be restarted if it is powered on. + if vmCtx.VM.Spec.NextRestartTime != "" { + + // If non-empty, the value of spec.nextRestartTime is guaranteed + // to be a valid RFC3339Nano timestamp due to the webhooks, + // however, we still check for the error due to testing that may + // not involve webhooks. + nextRestartTime, err := time.Parse( + time.RFC3339Nano, vmCtx.VM.Spec.NextRestartTime) + if err != nil { + return fmt.Errorf( + "spec.nextRestartTime %q cannot be parsed with %q %w", + vmCtx.VM.Spec.NextRestartTime, + time.RFC3339Nano, + err) + } + result, err := vmutil.RestartAndWait( + logr.NewContext(vmCtx, vmCtx.Logger), + vcVM.Client(), + vmutil.ManagedObjectFromObject(vcVM), + false, + nextRestartTime, + vmutil.ParsePowerOpMode(string(vmCtx.VM.Spec.RestartMode))) + if err != nil { + return err + } + if result.AnyChange() { + lastRestartTime := metav1.NewTime(nextRestartTime) + vmCtx.VM.Status.LastRestartTime = &lastRestartTime + } + } + + // Do not pass classConfigSpec to poweredOnVMReconfigure when VM is + // already powered on since we do not have to get VM class at this + // point. + return s.poweredOnVMReconfigure(vmCtx, resVM, config) + + case vmopv1.VirtualMachinePowerStateSuspended: + // A suspended VM cannot be reconfigured. + return resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmopv1.VirtualMachinePowerOpModeHard) + } + + updateArgs, err := getUpdateArgsFn() + if err != nil { + return err + } + + // TODO: Find a better place for this? + if err := s.attachClusterModule(vmCtx, resVM, updateArgs.ResourcePolicy); err != nil { + return err + } + + if err := s.prepareVMForPowerOn(vmCtx, resVM, config, updateArgs); err != nil { + return err + } + + if err := resVM.SetPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + existingPowerState, + vmCtx.VM.Spec.PowerState, + vmopv1.VirtualMachinePowerOpModeHard); err != nil { + return err + } + + if vmCtx.VM.Annotations == nil { + vmCtx.VM.Annotations = map[string]string{} + } + vmCtx.VM.Annotations[FirstBootDoneAnnotation] = "true" + } + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go new file mode 100644 index 000000000..bd3098660 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go @@ -0,0 +1,1236 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package session_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + vimTypes "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" +) + +var _ = Describe("Update ConfigSpec", func() { + + var ( + config *vimTypes.VirtualMachineConfigInfo + configSpec *vimTypes.VirtualMachineConfigSpec + ) + + BeforeEach(func() { + config = &vimTypes.VirtualMachineConfigInfo{} + configSpec = &vimTypes.VirtualMachineConfigSpec{} + }) + + // Just a few examples for testing these things here. Need to think more about whether this + // is a good way or not. Probably better to do this via UpdateVirtualMachine when we have + // better integration tests. + + Context("Basic Hardware", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + }) + + JustBeforeEach(func() { + session.UpdateHardwareConfigSpec(config, configSpec, vmClassSpec) + }) + + Context("Updates Hardware", func() { + BeforeEach(func() { + vmClassSpec.Hardware.Cpus = 42 + vmClassSpec.Hardware.Memory = resource.MustParse("2000Mi") + }) + + It("config spec is not empty", func() { + Expect(configSpec.NumCPUs).To(BeNumerically("==", 42)) + Expect(configSpec.MemoryMB).To(BeNumerically("==", 2000)) + }) + }) + + Context("config already matches", func() { + BeforeEach(func() { + config.Hardware.NumCPU = 42 + vmClassSpec.Hardware.Cpus = int64(config.Hardware.NumCPU) + config.Hardware.MemoryMB = 1500 + vmClassSpec.Hardware.Memory = resource.MustParse(fmt.Sprintf("%dMi", config.Hardware.MemoryMB)) + }) + + It("config spec show no changes", func() { + Expect(configSpec.NumCPUs).To(BeZero()) + Expect(configSpec.MemoryMB).To(BeZero()) + }) + }) + }) + + Context("CPU Allocation", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + var minCPUFreq uint64 = 1 + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + }) + + JustBeforeEach(func() { + session.UpdateConfigSpecCPUAllocation(config, configSpec, vmClassSpec, minCPUFreq) + }) + + It("config spec is empty", func() { + Expect(configSpec.CpuAllocation).To(BeNil()) + }) + + Context("config matches class policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Requests.Cpu = r + }) + + It("config spec is empty", func() { + Expect(configSpec.CpuAllocation).To(BeNil()) + }) + }) + + Context("config matches class policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Limits.Cpu = r + }) + + It("config spec is empty", func() { + Expect(configSpec.CpuAllocation).To(BeNil()) + }) + }) + + Context("config matches is different from policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(10 * virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Limits.Cpu = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.CpuAllocation.Reservation).To(BeNil()) + Expect(configSpec.CpuAllocation.Limit).ToNot(BeNil()) + Expect(*configSpec.CpuAllocation.Limit).To(BeNumerically("==", 100*1024*1024)) + }) + }) + + Context("config matches is different from policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.CpuAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(10 * virtualmachine.CPUQuantityToMhz(r, minCPUFreq)), + } + vmClassSpec.Policies.Resources.Requests.Cpu = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.CpuAllocation.Limit).To(BeNil()) + Expect(configSpec.CpuAllocation.Reservation).ToNot(BeNil()) + Expect(*configSpec.CpuAllocation.Reservation).To(BeNumerically("==", 100*1024*1024)) + }) + }) + }) + + Context("Memory Allocation", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + }) + + JustBeforeEach(func() { + session.UpdateConfigSpecMemoryAllocation(config, configSpec, vmClassSpec) + }) + + It("config spec is empty", func() { + Expect(configSpec.MemoryAllocation).To(BeNil()) + }) + + Context("config matches class policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Requests.Memory = r + }) + + It("config spec is empty", func() { + Expect(configSpec.MemoryAllocation).To(BeNil()) + }) + }) + + Context("config matches class policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Limits.Memory = r + }) + + It("config spec is empty", func() { + Expect(configSpec.MemoryAllocation).To(BeNil()) + }) + }) + + Context("config matches is different from policy limit", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Limit: pointer.Int64(10 * virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Limits.Memory = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation.Reservation).To(BeNil()) + Expect(configSpec.MemoryAllocation.Limit).ToNot(BeNil()) + Expect(*configSpec.MemoryAllocation.Limit).To(BeNumerically("==", 100)) + }) + }) + + Context("config matches is different from policy request", func() { + BeforeEach(func() { + r := resource.MustParse("100Mi") + config.MemoryAllocation = &vimTypes.ResourceAllocationInfo{ + Reservation: pointer.Int64(10 * virtualmachine.MemoryQuantityToMb(r)), + } + vmClassSpec.Policies.Resources.Requests.Memory = r + }) + + It("config spec is not empty", func() { + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation.Limit).To(BeNil()) + Expect(configSpec.MemoryAllocation.Reservation).ToNot(BeNil()) + Expect(*configSpec.MemoryAllocation.Reservation).To(BeNumerically("==", 100)) + }) + }) + }) + + Context("ExtraConfig", func() { + var vmClassSpec *vmopv1.VirtualMachineClassSpec + var classConfigSpec *vimTypes.VirtualMachineConfigSpec + var vm *vmopv1.VirtualMachine + var globalExtraConfig map[string]string + var ecMap map[string]string + var imageV1Alpha1Compatible bool + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + }, + } + globalExtraConfig = make(map[string]string) + classConfigSpec = nil + }) + + JustBeforeEach(func() { + session.UpdateConfigSpecExtraConfig( + config, + configSpec, + classConfigSpec, + vmClassSpec, + vm, + globalExtraConfig, + imageV1Alpha1Compatible) + + ecMap = make(map[string]string) + for _, ec := range configSpec.ExtraConfig { + if optionValue := ec.GetOptionValue(); optionValue != nil { + ecMap[optionValue.Key] = optionValue.Value.(string) + } + } + }) + + Context("Empty input", func() { + It("No changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("Updates configSpec.ExtraConfig", func() { + BeforeEach(func() { + config.ExtraConfig = append(config.ExtraConfig, &vimTypes.OptionValue{ + Key: constants.VMOperatorV1Alpha1ExtraConfigKey, Value: constants.VMOperatorV1Alpha1ConfigReady}) + globalExtraConfig["guestinfo.test"] = "test" + globalExtraConfig["global"] = "test" + imageV1Alpha1Compatible = true + }) + + It("Expected configSpec.ExtraConfig", func() { + By("VM Image compatible", func() { + Expect(ecMap).To(HaveKeyWithValue("guestinfo.vmservice.defer-cloud-init", "enabled")) + }) + + By("Global map", func() { + Expect(ecMap).To(HaveKeyWithValue("guestinfo.test", "test")) + Expect(ecMap).To(HaveKeyWithValue("global", "test")) + }) + }) + + Context("When VM uses metadata transport types other than CloudInit", func() { + BeforeEach(func() { + vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + }) + It("defer cloud-init extra config is enabled", func() { + Expect(ecMap).To(HaveKeyWithValue("guestinfo.vmservice.defer-cloud-init", "enabled")) + }) + }) + + Context("When VM uses CloudInit metadata transport type", func() { + BeforeEach(func() { + vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + }) + It("defer cloud-init extra config is not enabled", func() { + Expect(ecMap).ToNot(HaveKeyWithValue("guestinfo.vmservice.defer-cloud-init", "enabled")) + }) + }) + }) + + Context("ExtraConfig value already exists", func() { + BeforeEach(func() { + config.ExtraConfig = append(config.ExtraConfig, &vimTypes.OptionValue{Key: "foo", Value: "bar"}) + globalExtraConfig["foo"] = "bar" + }) + + It("No changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("InstanceStorage related tests", func() { + + Context("When InstanceStorage is NOT configured on VM", func() { + It("No Changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("When InstanceStorage is configured on VM", func() { + BeforeEach(func() { + vm.Spec.Volumes = append(vm.Spec.Volumes, vmopv1.VirtualMachineVolume{ + Name: "pvc-volume-1", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-volume-1", + }, + InstanceVolumeClaim: &vmopv1.InstanceVolumeClaimVolumeSource{ + StorageClass: "dummyStorageClass", + Size: resource.MustParse("256Gi"), + }, + }, + }, + }) + }) + + It("maintenance mode powerOff extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + }) + }) + }) + + Context("ThunderPciDevices related test", func() { + + Context("when virtual devices are not present", func() { + It("No Changes", func() { + Expect(ecMap).To(BeEmpty()) + }) + }) + + Context("when vGPU device is available", func() { + BeforeEach(func() { + vmClassSpec.Hardware.Devices = vmopv1.VirtualDevices{VGPUDevices: []vmopv1.VGPUDevice{ + { + ProfileName: "test-vgpu-profile", + }, + }} + }) + + It("maintenance mode powerOff extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + }) + + It("PCI passthru MMIO extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, constants.PCIPassthruMMIOSizeDefault)) + }) + + Context("when PCI passthru MMIO override annotation is set", func() { + BeforeEach(func() { + vm.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] = "12345" + }) + + It("PCI passthru MMIO extraConfig should be set to override annotation value", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, "12345")) + }) + }) + }) + + Context("when DDPIO device is available", func() { + BeforeEach(func() { + vmClassSpec.Hardware.Devices = vmopv1.VirtualDevices{DynamicDirectPathIODevices: []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 123, + DeviceID: 24, + CustomLabel: "", + }, + }} + }) + + It("maintenance mode powerOff extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + }) + + It("PCI passthru MMIO extraConfig should be added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, constants.PCIPassthruMMIOSizeDefault)) + }) + + Context("when PCI passthru MMIO override annotation is set", func() { + BeforeEach(func() { + vm.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] = "12345" + }) + + It("PCI passthru MMIO extraConfig should be set to override annotation value", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, "12345")) + }) + }) + }) + }) + + Context("when VM_Class_as_Config_DaynDate FSS is enabled", func() { + var oldVMClassAsConfigDaynDateFunc func() bool + const dummyKey = "dummy-key" + const dummyVal = "dummy-val" + + BeforeEach(func() { + oldVMClassAsConfigDaynDateFunc = lib.IsVMClassAsConfigFSSDaynDateEnabled + lib.IsVMClassAsConfigFSSDaynDateEnabled = func() bool { + return true + } + }) + + AfterEach(func() { + lib.IsVMClassAsConfigFSSDaynDateEnabled = oldVMClassAsConfigDaynDateFunc + }) + + Context("classConfigSpec extra config is not nil", func() { + BeforeEach(func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ExtraConfig: []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{ + Key: dummyKey + "-1", + Value: dummyVal + "-1", + }, + &vimTypes.OptionValue{ + Key: dummyKey + "-2", + Value: dummyVal + "-2", + }, + }, + } + config.ExtraConfig = append(config.ExtraConfig, &vimTypes.OptionValue{Key: "hello", Value: "world"}) + }) + It("vm extra config overlaps with global extra config", func() { + globalExtraConfig["hello"] = "world" + + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-1", dummyVal+"-1")) + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-2", dummyVal+"-2")) + Expect(ecMap).ToNot(HaveKeyWithValue("hello", "world")) + }) + + It("global extra config overlaps with class config spec - class config spec takes precedence", func() { + globalExtraConfig[dummyKey+"-1"] = dummyVal + "-3" + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-1", dummyVal+"-1")) + Expect(ecMap).To(HaveKeyWithValue(dummyKey+"-2", dummyVal+"-2")) + }) + + Context("class config spec has vGPU and DDPIO devices", func() { + BeforeEach(func() { + classConfigSpec.DeviceChange = []vimTypes.BaseVirtualDeviceConfigSpec{ + &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimTypes.VirtualPCIPassthrough{ + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "SampleProfile2", + }, + }, + }, + }, + &vimTypes.VirtualDeviceConfigSpec{ + Operation: vimTypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimTypes.VirtualPCIPassthrough{ + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "SampleLabel2", + }, + }, + }, + }, + } + + }) + + It("extraConfig Map has MMIO and MMPowerOff related keys added", func() { + Expect(ecMap).To(HaveKeyWithValue(constants.MMPowerOffVMExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOExtraConfigKey, constants.ExtraConfigTrue)) + Expect(ecMap).To(HaveKeyWithValue(constants.PCIPassthruMMIOSizeExtraConfigKey, constants.PCIPassthruMMIOSizeDefault)) + }) + }) + }) + }) + }) + + Context("ChangeBlockTracking", func() { + var vmSpec vmopv1.VirtualMachineSpec + var classConfigSpec *vimTypes.VirtualMachineConfigSpec + + BeforeEach(func() { + config.ChangeTrackingEnabled = nil + classConfigSpec = nil + }) + + AfterEach(func() { + configSpec.ChangeTrackingEnabled = nil + }) + + It("cbt and status cbt unset", func() { + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).To(BeNil()) + }) + + It("configSpec cbt set to true OMG", func() { + config.ChangeTrackingEnabled = pointer.Bool(true) + vmSpec.Advanced.ChangeBlockTracking = false + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeFalse()) + }) + + It("configSpec cbt set to false", func() { + config.ChangeTrackingEnabled = pointer.Bool(false) + vmSpec.Advanced.ChangeBlockTracking = true + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeTrue()) + }) + + It("configSpec cbt matches", func() { + config.ChangeTrackingEnabled = pointer.Bool(true) + vmSpec.Advanced.ChangeBlockTracking = true + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).To(BeNil()) + }) + + It("classConfigSpec not nil and is ignored", func() { + config.ChangeTrackingEnabled = pointer.Bool(false) + vmSpec.Advanced.ChangeBlockTracking = true + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ChangeTrackingEnabled: pointer.Bool(false), + } + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeTrue()) + }) + + Context("VM_Class_as_Config_DaynDate FSS is enabled", func() { + var oldVMClassAsConfigDaynDateFunc func() bool + BeforeEach(func() { + oldVMClassAsConfigDaynDateFunc = lib.IsVMClassAsConfigFSSDaynDateEnabled + lib.IsVMClassAsConfigFSSDaynDateEnabled = func() bool { + return true + } + config.ChangeTrackingEnabled = pointer.Bool(false) + vmSpec.Advanced.ChangeBlockTracking = true + }) + + AfterEach(func() { + lib.IsVMClassAsConfigFSSDaynDateEnabled = oldVMClassAsConfigDaynDateFunc + }) + + It("classConfigSpec not nil and same as configInfo", func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ChangeTrackingEnabled: pointer.Bool(false), + } + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).To(BeNil()) + }) + + It("classConfigSpec not nil, different from configInfo, overrides vm spec cbt", func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{ + ChangeTrackingEnabled: pointer.Bool(true), + } + + session.UpdateConfigSpecChangeBlockTracking(config, configSpec, classConfigSpec, vmSpec) + Expect(configSpec.ChangeTrackingEnabled).ToNot(BeNil()) + Expect(*configSpec.ChangeTrackingEnabled).To(BeTrue()) + }) + }) + }) + + Context("Firmware", func() { + var vm *vmopv1.VirtualMachine + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: make(map[string]string), + }, + } + config.Firmware = "bios" + }) + + It("No firmware annotation", func() { + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(BeEmpty()) + }) + + It("Set firmware annotation equal to current vm firmware", func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = config.Firmware + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(BeEmpty()) + }) + + It("Set firmware annotation differing to current vm firmware", func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = "efi" + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(Equal("efi")) + }) + + It("Set firmware annotation to an invalid value", func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = "invalidfirmware" + session.UpdateConfigSpecFirmware(config, configSpec, vm) + Expect(configSpec.Firmware).To(BeEmpty()) + }) + }) + + Context("DeviceGroups", func() { + var classConfigSpec *vimTypes.VirtualMachineConfigSpec + + BeforeEach(func() { + classConfigSpec = &vimTypes.VirtualMachineConfigSpec{} + }) + + It("No DeviceGroups set in class config spec", func() { + session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) + Expect(configSpec.DeviceGroups).To(BeNil()) + }) + + It("DeviceGroups set in class config spec", func() { + classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ + DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ + &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ + GroupInstanceKey: int32(400), + }, + }, + } + + session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) + Expect(configSpec.DeviceGroups).NotTo(BeNil()) + Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) + deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() + Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) + }) + + It("configInfo DeviceGroups set with vals different than the class config spec", func() { + classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ + DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ + &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ + GroupInstanceKey: int32(400), + }, + }, + } + + config.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ + DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ + &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ + GroupInstanceKey: int32(500), + }, + }, + } + + session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) + Expect(configSpec.DeviceGroups).NotTo(BeNil()) + Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) + deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() + Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) + }) + }) + + Context("Ethernet Card Changes", func() { + var expectedList object.VirtualDeviceList + var currentList object.VirtualDeviceList + var deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + var dvpg1 *vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo + var dvpg2 *vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo + var err error + + BeforeEach(func() { + dvpg1 = &vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo{ + Port: vimTypes.DistributedVirtualSwitchPortConnection{ + PortgroupKey: "key1", + SwitchUuid: "uuid1", + }, + } + + dvpg2 = &vimTypes.VirtualEthernetCardDistributedVirtualPortBackingInfo{ + Port: vimTypes.DistributedVirtualSwitchPortConnection{ + PortgroupKey: "key2", + SwitchUuid: "uuid2", + }, + } + }) + + JustBeforeEach(func() { + deviceChanges, err = session.UpdateEthCardDeviceChanges(expectedList, currentList) + }) + + AfterEach(func() { + currentList = nil + expectedList = nil + }) + + Context("No devices", func() { + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + + Context("No device change when nothing changes", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + currentList = append(currentList, card2) + }) + + It("returns no device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(0)) + }) + }) + + Context("Add device", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + }) + + It("returns add device change", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(1)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Add and remove device when backing change", func() { + var card1 vimTypes.BaseVirtualDevice + var card2 vimTypes.BaseVirtualDevice + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg2) + Expect(err).ToNot(HaveOccurred()) + currentList = append(currentList, card2) + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Add and remove device when MAC address is different", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + card1.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().AddressType = string(vimTypes.VirtualEthernetCardMacTypeManual) + card1.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().MacAddress = "mac1" + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + card2.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().AddressType = string(vimTypes.VirtualEthernetCardMacTypeManual) + card2.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().MacAddress = "mac2" + currentList = append(currentList, card2) + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("When WCP_VMClass_as_Config is enabled, Add and remove device when card type is different", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + var oldVMClassAsConfigFunc func() bool + + BeforeEach(func() { + oldVMClassAsConfigFunc = lib.IsVMClassAsConfigFSSDaynDateEnabled + lib.IsVMClassAsConfigFSSDaynDateEnabled = func() bool { + return true + } + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet2", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + currentList = append(currentList, card2) + }) + + AfterEach(func() { + lib.IsVMClassAsConfigFSSDaynDateEnabled = oldVMClassAsConfigFunc + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Add and remove device when ExternalID is different", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + card1.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().ExternalId = "ext1" + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + card2.(vimTypes.BaseVirtualEthernetCard).GetVirtualEthernetCard().ExternalId = "ext2" + currentList = append(currentList, card2) + }) + + It("returns remove and add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(HaveLen(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card2.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(card1.GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("Keeps existing device with same backing", func() { + var card1 vimTypes.BaseVirtualDevice + var key1 int32 = 100 + var card2 vimTypes.BaseVirtualDevice + var key2 int32 = 200 + + BeforeEach(func() { + card1, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card1.GetVirtualDevice().Key = key1 + expectedList = append(expectedList, card1) + + card2, err = object.EthernetCardTypes().CreateEthernetCard("vmxnet3", dvpg1) + Expect(err).ToNot(HaveOccurred()) + card2.GetVirtualDevice().Key = key2 + currentList = append(currentList, card2) + }) + + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + }) + + Context("Create vSphere PCI device", func() { + var vgpuDevices = []vmopv1.VGPUDevice{ + { + ProfileName: "SampleProfile", + }, + } + var ddpioDevices = []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 42, + DeviceID: 43, + CustomLabel: "SampleLabel", + }, + } + var pciDevices vmopv1.VirtualDevices + Context("For VM Class Spec vGPU device", func() { + BeforeEach(func() { + pciDevices = vmopv1.VirtualDevices{ + VGPUDevices: vgpuDevices, + } + }) + It("should create vSphere device with VmiopBackingInfo", func() { + vSphereDevices := virtualmachine.CreatePCIDevicesFromVMClass(pciDevices) + Expect(vSphereDevices).To(HaveLen(1)) + virtualDevice := vSphereDevices[0].GetVirtualDevice() + backing := virtualDevice.Backing.(*vimTypes.VirtualPCIPassthroughVmiopBackingInfo) + Expect(backing.Vgpu).To(Equal(pciDevices.VGPUDevices[0].ProfileName)) + }) + }) + Context("For VM Class Spec Dynamic DirectPath I/O device", func() { + BeforeEach(func() { + pciDevices = vmopv1.VirtualDevices{ + DynamicDirectPathIODevices: ddpioDevices, + } + }) + It("should create vSphere device with DynamicBackingInfo", func() { + vSphereDevices := virtualmachine.CreatePCIDevicesFromVMClass(pciDevices) + Expect(vSphereDevices).To(HaveLen(1)) + virtualDevice := vSphereDevices[0].GetVirtualDevice() + backing := virtualDevice.Backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo) + Expect(backing.AllowedDevice[0].DeviceId).To(Equal(int32(pciDevices.DynamicDirectPathIODevices[0].DeviceID))) + Expect(backing.AllowedDevice[0].VendorId).To(Equal(int32(pciDevices.DynamicDirectPathIODevices[0].VendorID))) + Expect(backing.CustomLabel).To(Equal(pciDevices.DynamicDirectPathIODevices[0].CustomLabel)) + }) + }) + + When("PCI devices from ConfigSpec are specified", func() { + + var devIn []*vimTypes.VirtualPCIPassthrough + + Context("For ConfigSpec VGPU device", func() { + BeforeEach(func() { + devIn = []*vimTypes.VirtualPCIPassthrough{ + { + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "configspec-profile", + }, + }, + }, + } + }) + It("should create vSphere device with VmiopBackingInfo", func() { + devList := virtualmachine.CreatePCIDevicesFromConfigSpec(devIn) + Expect(devList).To(HaveLen(1)) + + Expect(devList[0]).ToNot(BeNil()) + Expect(devList[0]).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthrough{})) + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing).ToNot(BeNil()) + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthroughVmiopBackingInfo{})) + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing.(*vimTypes.VirtualPCIPassthroughVmiopBackingInfo).Vgpu).To(Equal("configspec-profile")) + }) + }) + + Context("For ConfigSpec DirectPath I/O device", func() { + BeforeEach(func() { + devIn = []*vimTypes.VirtualPCIPassthrough{ + { + VirtualDevice: vimTypes.VirtualDevice{ + Backing: &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + CustomLabel: "configspec-ddpio-label", + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 456, + DeviceId: 457, + }, + }, + }, + }, + }, + } + }) + It("should create vSphere device with DynamicBackingInfo", func() { + devList := virtualmachine.CreatePCIDevicesFromConfigSpec(devIn) + Expect(devList).To(HaveLen(1)) + + Expect(devList[0]).ToNot(BeNil()) + Expect(devList[0]).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthrough{})) + + Expect(devList[0].(*vimTypes.VirtualPCIPassthrough).Backing).ToNot(BeNil()) + backing := devList[0].(*vimTypes.VirtualPCIPassthrough).Backing + Expect(backing).To(BeAssignableToTypeOf(&vimTypes.VirtualPCIPassthroughDynamicBackingInfo{})) + + Expect(backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo).CustomLabel).To(Equal("configspec-ddpio-label")) + Expect(backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo).AllowedDevice[0].VendorId).To(BeEquivalentTo(456)) + Expect(backing.(*vimTypes.VirtualPCIPassthroughDynamicBackingInfo).AllowedDevice[0].DeviceId).To(BeEquivalentTo(457)) + }) + }) + }) + }) + + Context("PCI Device Changes", func() { + var ( + currentList, expectedList object.VirtualDeviceList + deviceChanges []vimTypes.BaseVirtualDeviceConfigSpec + err error + + // Variables related to vGPU devices. + backingInfo1, backingInfo2 *vimTypes.VirtualPCIPassthroughVmiopBackingInfo + deviceKey1, deviceKey2 int32 + vGPUDevice1, vGPUDevice2 vimTypes.BaseVirtualDevice + + // Variables related to dynamicDirectPathIO devices. + allowedDev1, allowedDev2 vimTypes.VirtualPCIPassthroughAllowedDevice + backingInfo3, backingInfo4 *vimTypes.VirtualPCIPassthroughDynamicBackingInfo + deviceKey3, deviceKey4 int32 + dynamicDirectPathIODev1, dynamicDirectPathIODev2 vimTypes.BaseVirtualDevice + ) + + BeforeEach(func() { + backingInfo1 = &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{Vgpu: "mockup-vmiop1"} + backingInfo2 = &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{Vgpu: "mockup-vmiop2"} + deviceKey1 = int32(-200) + deviceKey2 = int32(-201) + vGPUDevice1 = virtualmachine.CreatePCIPassThroughDevice(deviceKey1, backingInfo1) + vGPUDevice2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey2, backingInfo2) + + allowedDev1 = vimTypes.VirtualPCIPassthroughAllowedDevice{ + VendorId: 1000, + DeviceId: 100, + } + allowedDev2 = vimTypes.VirtualPCIPassthroughAllowedDevice{ + VendorId: 2000, + DeviceId: 200, + } + backingInfo3 = &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{allowedDev1}, + CustomLabel: "sampleLabel3", + } + backingInfo4 = &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{allowedDev2}, + CustomLabel: "sampleLabel4", + } + deviceKey3 = int32(-202) + deviceKey4 = int32(-203) + dynamicDirectPathIODev1 = virtualmachine.CreatePCIPassThroughDevice(deviceKey3, backingInfo3) + dynamicDirectPathIODev2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey4, backingInfo4) + }) + + JustBeforeEach(func() { + deviceChanges, err = session.UpdatePCIDeviceChanges(expectedList, currentList) + }) + + AfterEach(func() { + currentList = nil + expectedList = nil + }) + + Context("No devices", func() { + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + + Context("Adding vGPU and dynamicDirectPathIO devices with different backing info", func() { + BeforeEach(func() { + expectedList = append(expectedList, vGPUDevice1) + expectedList = append(expectedList, vGPUDevice2) + expectedList = append(expectedList, dynamicDirectPathIODev1) + expectedList = append(expectedList, dynamicDirectPathIODev2) + }) + + It("Should return add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(len(expectedList))) + + for idx, dev := range deviceChanges { + configSpec := dev.GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[idx].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + } + }) + }) + + Context("Adding vGPU and dynamicDirectPathIO devices with same backing info", func() { + BeforeEach(func() { + expectedList = append(expectedList, vGPUDevice1) + // Creating a vGPUDevice with same backingInfo1 but different deviceKey. + vGPUDevice2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey2, backingInfo1) + expectedList = append(expectedList, vGPUDevice2) + expectedList = append(expectedList, dynamicDirectPathIODev1) + // Creating a dynamicDirectPathIO device with same backingInfo3 but different deviceKey. + dynamicDirectPathIODev2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey4, backingInfo3) + expectedList = append(expectedList, dynamicDirectPathIODev2) + }) + + It("Should return add device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(len(expectedList))) + + for idx, dev := range deviceChanges { + configSpec := dev.GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[idx].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + } + }) + }) + + Context("When the expected and current lists have DDPIO devices with different custom labels", func() { + BeforeEach(func() { + expectedList = []vimTypes.BaseVirtualDevice{dynamicDirectPathIODev1} + // Creating a dynamicDirectPathIO device with same backing info except for the custom label. + backingInfoDiffCustomLabel := &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: backingInfo3.AllowedDevice, + CustomLabel: "DifferentLabel", + } + dynamicDirectPathIODev2 = virtualmachine.CreatePCIPassThroughDevice(deviceKey4, backingInfoDiffCustomLabel) + currentList = []vimTypes.BaseVirtualDevice{dynamicDirectPathIODev2} + }) + + It("should return add and remove device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(2)) + + configSpec := deviceChanges[0].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(currentList[0].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + + configSpec = deviceChanges[1].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[0].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + }) + }) + + Context("When the expected and current list of pciDevices have different Devices", func() { + BeforeEach(func() { + currentList = append(currentList, vGPUDevice1) + expectedList = append(expectedList, vGPUDevice2) + currentList = append(currentList, dynamicDirectPathIODev1) + expectedList = append(expectedList, dynamicDirectPathIODev2) + }) + + It("Should return add and remove device changes", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(len(deviceChanges)).To(Equal(4)) + + for i := 0; i < 2; i++ { + configSpec := deviceChanges[i].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(currentList[i].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationRemove)) + } + + for i := 2; i < 4; i++ { + configSpec := deviceChanges[i].GetVirtualDeviceConfigSpec() + Expect(configSpec.Device.GetVirtualDevice().Key).To(Equal(expectedList[i-2].GetVirtualDevice().Key)) + Expect(configSpec.Operation).To(Equal(vimTypes.VirtualDeviceConfigSpecOperationAdd)) + } + }) + }) + + Context("When the expected and current list of pciDevices have same Devices", func() { + BeforeEach(func() { + currentList = append(currentList, vGPUDevice1) + expectedList = append(expectedList, vGPUDevice1) + currentList = append(currentList, dynamicDirectPathIODev1) + expectedList = append(expectedList, dynamicDirectPathIODev1) + }) + + It("returns empty list", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(deviceChanges).To(BeEmpty()) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/storage/provisioning.go b/pkg/vmprovider/providers/vsphere2/storage/provisioning.go new file mode 100644 index 000000000..5a3997a65 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/storage/provisioning.go @@ -0,0 +1,88 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "github.com/pkg/errors" + "github.com/vmware/govmomi/pbm" + pbmTypes "github.com/vmware/govmomi/pbm/types" + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" +) + +// getProfileProportionalCapacity returns the storage profile "proportionalCapacity" value if the +// policy is for vSAN. The proportionalCapacity is percentage of the logical size of the storage +// object that will be reserved upon provisioning. Returns -1 if not specified. +// The UI presents options for "thin" (0%), 25%, 50%, 75% and "thick" (100%). +func getProfileProportionalCapacity(profile pbmTypes.BasePbmProfile) int32 { + capProfile, ok := profile.(*pbmTypes.PbmCapabilityProfile) + if !ok { + return -1 + } + + if capProfile.ResourceType.ResourceType != string(pbmTypes.PbmProfileResourceTypeEnumSTORAGE) { + return -1 + } + + if capProfile.ProfileCategory != string(pbmTypes.PbmProfileCategoryEnumREQUIREMENT) { + return -1 + } + + sub, ok := capProfile.Constraints.(*pbmTypes.PbmCapabilitySubProfileConstraints) + if !ok { + return -1 + } + + for _, p := range sub.SubProfiles { + for _, capability := range p.Capability { + if capability.Id.Namespace != "VSAN" || capability.Id.Id != "proportionalCapacity" { + continue + } + + for _, c := range capability.Constraint { + for _, prop := range c.PropertyInstance { + if prop.Id != capability.Id.Id { + continue + } + if val, ok := prop.Value.(int32); ok { + return val + } + } + } + } + } + + return -1 +} + +// GetDiskProvisioningForProfile returns the provisioning type for the storage profile if it has +// one specified. +func GetDiskProvisioningForProfile( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + storageProfileID string) (string, error) { + + c, err := pbm.NewClient(vmCtx, vcClient.VimClient()) + if err != nil { + return "", err + } + + profiles, err := c.RetrieveContent(vmCtx, []pbmTypes.PbmProfileId{{UniqueId: storageProfileID}}) + if err != nil { + return "", errors.Wrapf(err, "Failed to get storage profiles for ID: %s", storageProfileID) + } + + for _, p := range profiles { + switch getProfileProportionalCapacity(p) { + case 0: + return string(vimTypes.OvfCreateImportSpecParamsDiskProvisioningTypeThin), nil + case 100: + return string(vimTypes.OvfCreateImportSpecParamsDiskProvisioningTypeThick), nil + } + } + + return "", nil +} diff --git a/pkg/vmprovider/providers/vsphere2/storage/storageclass.go b/pkg/vmprovider/providers/vsphere2/storage/storageclass.go new file mode 100644 index 000000000..511e651a0 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/storage/storageclass.go @@ -0,0 +1,83 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "fmt" + + storagev1 "k8s.io/api/storage/v1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +// GetStoragePolicyID returns Storage Policy ID from Storage Class Name. +func GetStoragePolicyID( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client, + storageClassName string) (string, error) { + + sc := &storagev1.StorageClass{} + if err := client.Get(vmCtx, ctrlclient.ObjectKey{Name: storageClassName}, sc); err != nil { + vmCtx.Logger.Error(err, "Failed to get StorageClass", "storageClass", storageClassName) + return "", err + } + + policyID, ok := sc.Parameters["storagePolicyID"] + if !ok { + return "", fmt.Errorf("StorageClass %s does not have 'storagePolicyID' parameter", storageClassName) + } + + return policyID, nil +} + +// GetVMStoragePoliciesIDs returns a map of storage class names to their storage policy IDs. +func GetVMStoragePoliciesIDs( + vmCtx context.VirtualMachineContextA2, + client ctrlclient.Client) (map[string]string, error) { + + storageClassNames := getVMStorageClassNames(vmCtx.VM) + storageClassesToIDs := map[string]string{} + + for _, name := range storageClassNames { + if _, ok := storageClassesToIDs[name]; !ok { + id, err := GetStoragePolicyID(vmCtx, client, name) + if err != nil { + return nil, err + } + + storageClassesToIDs[name] = id + } + } + + return storageClassesToIDs, nil +} + +func getVMStorageClassNames(vm *vmopv1.VirtualMachine) []string { + var names []string + + if vm.Spec.StorageClass != "" { + names = append(names, vm.Spec.StorageClass) + } + + for _, vol := range vm.Spec.Volumes { + var storageClass string + + claim := vol.PersistentVolumeClaim + if claim != nil { + if isClaim := claim.InstanceVolumeClaim; isClaim != nil { + storageClass = isClaim.StorageClass + } else { //nolint + // TODO: Fetch claim.ClaimName PVC to get the StorageClass. + } + } + + if storageClass != "" { + names = append(names, storageClass) + } + } + + return names +} diff --git a/pkg/vmprovider/providers/vsphere2/test/pki.go b/pkg/vmprovider/providers/vsphere2/test/pki.go new file mode 100644 index 000000000..557569440 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/test/pki.go @@ -0,0 +1,70 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "time" + + . "github.com/onsi/gomega" +) + +func GeneratePrivateKey() *rsa.PrivateKey { + reader := rand.Reader + bitSize := 2048 + + // Based on https://golang.org/src/crypto/tls/generate_cert.go + privateKey, err := rsa.GenerateKey(reader, bitSize) + if err != nil { + panic("failed to generate private key") + } + return privateKey +} + +func GenerateSelfSignedCert() (string, string) { + priv := GeneratePrivateKey() + now := time.Now() + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + Expect(err).NotTo(HaveOccurred()) + + template := x509.Certificate{ + Subject: pkix.Name{ + Organization: []string{"Acme Co"}, + }, + SerialNumber: serialNumber, + NotBefore: now, + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + template.IPAddresses = []net.IP{net.ParseIP("127.0.0.1")} + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + Expect(err).NotTo(HaveOccurred()) + certOut, err := os.CreateTemp("", "cert.pem") + Expect(err).NotTo(HaveOccurred()) + err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + Expect(err).NotTo(HaveOccurred()) + err = certOut.Close() + Expect(err).NotTo(HaveOccurred()) + + keyOut, err := os.CreateTemp("", "key.pem") + Expect(err).NotTo(HaveOccurred()) + privBytes, err := x509.MarshalPKCS8PrivateKey(priv) + Expect(err).NotTo(HaveOccurred()) + err = pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) + Expect(err).NotTo(HaveOccurred()) + err = keyOut.Close() + Expect(err).NotTo(HaveOccurred()) + + return keyOut.Name(), certOut.Name() +} diff --git a/pkg/vmprovider/providers/vsphere2/test/suite.go b/pkg/vmprovider/providers/vsphere2/test/suite.go new file mode 100644 index 000000000..cf496c076 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/test/suite.go @@ -0,0 +1,61 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "context" + "crypto/tls" + "os" + + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" + _ "github.com/vmware/govmomi/vapi/simulator" // blank import the VAPI simulator bindings +) + +func BeforeSuite() (ctx context.Context, + model *simulator.Model, + server *simulator.Server, + tlsKeyPath, tlsCertPath string, + tlsModel *simulator.Model, + tlsServer *simulator.Server) { + + ctx = context.Background() + + // Set up a simulator for testing most client interactions (ignoring TLS) + model, server = SetupModelAndServerWithSettings(&tls.Config{ + MinVersion: tls.VersionTLS12, + }) + + // Set up a second simulator for testing TLS. + tlsKeyPath, tlsCertPath = GenerateSelfSignedCert() + tlsCert, err := tls.LoadX509KeyPair(tlsCertPath, tlsKeyPath) + Expect(err).NotTo(HaveOccurred()) + tlsModel, tlsServer = SetupModelAndServerWithSettings(&tls.Config{ + Certificates: []tls.Certificate{ + tlsCert, + }, + PreferServerCipherSuites: true, + MinVersion: tls.VersionTLS12, + }) + + return +} + +func AfterSuite( + ctx context.Context, + model *simulator.Model, + server *simulator.Server, + tlsKeyPath, tlsCertPath string, + tlsModel *simulator.Model, + tlsServer *simulator.Server) { + + server.Close() + model.Remove() + + tlsServer.Close() + tlsModel.Remove() + + _ = os.Remove(tlsKeyPath) + _ = os.Remove(tlsCertPath) +} diff --git a/pkg/vmprovider/providers/vsphere2/test/vcsim.go b/pkg/vmprovider/providers/vsphere2/test/vcsim.go new file mode 100644 index 000000000..607ed81e2 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/test/vcsim.go @@ -0,0 +1,31 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "crypto/tls" + + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/simulator" +) + +func SetupModelAndServerWithSettings(tlsConfig *tls.Config) (*simulator.Model, *simulator.Server) { + newModel := simulator.VPX() + + // By Default, the Model being used by vcsim has two ResourcePools + // (one for the cluster and host each). Setting Model.Host=0 ensures + // we only have one ResourcePool, making it easier to pick the + // ResourcePool without having to look up using a hardcoded path. + newModel.Host = 0 + + err := newModel.Create() + Expect(err).ToNot(HaveOccurred()) + + newModel.Service.RegisterEndpoints = true + + newModel.Service.TLS = tlsConfig + newServer := newModel.Service.NewServer() + + return newModel, newServer +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/cluster.go b/pkg/vmprovider/providers/vsphere2/vcenter/cluster.go new file mode 100644 index 000000000..2cbee7adb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/cluster.go @@ -0,0 +1,44 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + goctx "context" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25/mo" +) + +// ClusterMinCPUFreq returns the minimum frequency across all the hosts in the cluster. This is needed to +// convert the CPU requirements specified in cores to MHz. vSphere core is assumed to be equivalent to the +// value of min frequency. This function is adapted from wcp schedext. +func ClusterMinCPUFreq(ctx goctx.Context, cluster *object.ClusterComputeResource) (uint64, error) { + var cr mo.ComputeResource + if err := cluster.Properties(ctx, cluster.Reference(), []string{"host"}, &cr); err != nil { + return 0, err + } + + if len(cr.Host) == 0 { + return 0, nil + } + + var hosts []mo.HostSystem + pc := property.DefaultCollector(cluster.Client()) + if err := pc.Retrieve(ctx, cr.Host, []string{"summary"}, &hosts); err != nil { + return 0, err + } + + var minFreq uint64 + for _, h := range hosts { + if hw := h.Summary.Hardware; hw != nil { + hostCPUMHz := uint64(hw.CpuMhz) + if hostCPUMHz < minFreq || minFreq == 0 { + minFreq = hostCPUMHz + } + } + } + + return minFreq, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go new file mode 100644 index 000000000..5b90b5145 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/cluster_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func clusterTests() { + Describe("ClusterMinCPUFreq", minFreq) +} + +func minFreq() { + // Hardcoded value in govmomi simulator/esx/host_system.go + const expectedCPUFreq = 2294 + + var ( + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Describe("ClusterMinCPUFreq", func() { + It("returns min freq of hosts in cluster", func() { + cpuFreq, err := vcenter.ClusterMinCPUFreq(ctx, ctx.GetSingleClusterCompute()) + Expect(err).ToNot(HaveOccurred()) + Expect(cpuFreq).Should(BeEquivalentTo(expectedCPUFreq)) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/folder.go b/pkg/vmprovider/providers/vsphere2/vcenter/folder.go new file mode 100644 index 000000000..414a08a67 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/folder.go @@ -0,0 +1,138 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + goctx "context" + "fmt" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" +) + +// GetFolderByMoID returns the vim Folder for the MoID. +func GetFolderByMoID( + ctx goctx.Context, + finder *find.Finder, + folderMoID string) (*object.Folder, error) { + + o, err := finder.ObjectReference(ctx, types.ManagedObjectReference{Type: "Folder", Value: folderMoID}) + if err != nil { + return nil, err + } + + return o.(*object.Folder), nil +} + +// GetChildFolder gets the named child Folder from the parent Folder. +func GetChildFolder( + ctx goctx.Context, + parentFolder *object.Folder, + childName string) (*object.Folder, error) { + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil { + return nil, err + } else if childFolder == nil { + return nil, fmt.Errorf("folder child %s not found under parent Folder %s", + childName, parentFolder.Reference().Value) + } + + return childFolder, nil +} + +// DoesChildFolderExist returns if the named child Folder exists under the parent Folder. +func DoesChildFolderExist( + ctx goctx.Context, + vimClient *vim25.Client, + parentFolderMoID, childName string) (bool, error) { + + parentFolder := object.NewFolder(vimClient, + types.ManagedObjectReference{Type: "Folder", Value: parentFolderMoID}) + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil { + return false, err + } + + return childFolder != nil, nil +} + +// CreateFolder creates the named child Folder under the parent Folder. +func CreateFolder( + ctx goctx.Context, + vimClient *vim25.Client, + parentFolderMoID, childName string) (string, error) { + + parentFolder := object.NewFolder(vimClient, + types.ManagedObjectReference{Type: "Folder", Value: parentFolderMoID}) + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil { + return "", err + } + + if childFolder == nil { + folder, err := parentFolder.CreateFolder(ctx, childName) + if err != nil { + return "", err + } + + childFolder = folder + } + + return childFolder.Reference().Value, nil +} + +// DeleteChildFolder deletes the child Folder under the parent Folder. +func DeleteChildFolder( + ctx goctx.Context, + vimClient *vim25.Client, + parentFolderMoID, childName string) error { + + parentFolder := object.NewFolder(vimClient, + types.ManagedObjectReference{Type: "Folder", Value: parentFolderMoID}) + + childFolder, err := findChildFolder(ctx, parentFolder, childName) + if err != nil || childFolder == nil { + return err + } + + task, err := childFolder.Destroy(ctx) + if err != nil { + return err + } + + if taskResult, err := task.WaitForResult(ctx); err != nil { + if taskResult == nil || taskResult.Error == nil { + return err + } + return fmt.Errorf("destroy Folder %s task failed: %w: %s", + childFolder.Reference().Value, err, taskResult.Error.LocalizedMessage) + } + + return nil +} + +func findChildFolder( + ctx goctx.Context, + parentFolder *object.Folder, + childName string) (*object.Folder, error) { + + objRef, err := object.NewSearchIndex(parentFolder.Client()).FindChild(ctx, parentFolder.Reference(), childName) + if err != nil { + return nil, err + } else if objRef == nil { + return nil, nil + } + + folder, ok := objRef.(*object.Folder) + if !ok { + return nil, fmt.Errorf("Folder child %q is not Folder but a %T", childName, objRef) //nolint + } + + return folder, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go new file mode 100644 index 000000000..99cfa9426 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/folder_test.go @@ -0,0 +1,173 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func folderTests() { + Describe("GetFolderByMoID", getFolderByMoID) + Describe("CreateDeleteExistsFolder", createDeleteExistsFolder) +} + +func getFolderByMoID() { + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("returns success", func() { + moID := nsInfo.Folder.Reference().Value + + folder, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, moID) + Expect(err).ToNot(HaveOccurred()) + Expect(folder).ToNot(BeNil()) + Expect(folder.Name()).To(Equal(nsInfo.Namespace)) + }) + + It("returns error when moID does not exist", func() { + folder, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, "bogus") + Expect(err).To(HaveOccurred()) + Expect(folder).To(BeNil()) + }) +} + +func createDeleteExistsFolder() { + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + + parentFolderMoID string + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + parentFolderMoID = nsInfo.Folder.Reference().Value + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + parentFolderMoID = "" + }) + + Context("CreateFolder", func() { + It("creates child Folder", func() { + childMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(childMoID).ToNot(BeEmpty()) + + By("NoOp when child Folder already exists", func() { + moID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(moID).To(Equal(childMoID)) + }) + + By("child Folder is found by MoID", func() { + folder, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, childMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(folder.Reference().Value).To(Equal(childMoID)) + }) + }) + + It("returns error when parent Folder MoID does not exist", func() { + childMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, "bogus", "myFolder") + Expect(err).To(HaveOccurred()) + Expect(childMoID).To(BeEmpty()) + }) + }) + + Context("GetChildFolder", func() { + It("returns success when child Folder exists", func() { + childFolderMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + parentFolder := object.NewFolder(ctx.VCClient.Client, types.ManagedObjectReference{ + Type: "Folder", + Value: parentFolderMoID, + }) + + childFolder, err := vcenter.GetChildFolder(ctx, parentFolder, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(childFolder).ToNot(BeNil()) + Expect(childFolder.Reference().Value).To(Equal(childFolderMoID)) + }) + + It("returns error when child Folder does not exists", func() { + parentFolder := object.NewFolder(ctx.VCClient.Client, types.ManagedObjectReference{ + Type: "Folder", + Value: parentFolderMoID, + }) + + childFolder, err := vcenter.GetChildFolder(ctx, parentFolder, "myFolder") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found under parent Folder")) + Expect(childFolder).To(BeNil()) + }) + }) + + Context("DoesChildFolderExist", func() { + It("returns true when child Folder exists", func() { + _, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + exists, err := vcenter.DoesChildFolderExist(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + + It("returns false when child Folder does not exist", func() { + exists, err := vcenter.DoesChildFolderExist(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + + It("returns error when parent Folder MoID does not exist", func() { + exists, err := vcenter.DoesChildFolderExist(ctx, ctx.VCClient.Client, "bogus", "myFolder") + Expect(err).To(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("DeleteFolder", func() { + It("deletes child Folder", func() { + childMoID, err := vcenter.CreateFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + err = vcenter.DeleteChildFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + + By("child Folder is not found by MoID", func() { + _, err := vcenter.GetFolderByMoID(ctx, ctx.Finder, childMoID) + Expect(err).To(HaveOccurred()) + }) + + By("NoOp when child does not exist", func() { + err := vcenter.DeleteChildFolder(ctx, ctx.VCClient.Client, parentFolderMoID, "myFolder") + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/getvm.go b/pkg/vmprovider/providers/vsphere2/vcenter/getvm.go new file mode 100644 index 000000000..f6363a31c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/getvm.go @@ -0,0 +1,152 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "fmt" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/topology" +) + +// GetVirtualMachine gets the VM from VC, either by the MoID, UUID, or the inventory path. +func GetVirtualMachine( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client, + vimClient *vim25.Client, + datacenter *object.Datacenter, + finder *find.Finder) (*object.VirtualMachine, error) { + + if uniqueID := vmCtx.VM.Status.UniqueID; uniqueID != "" { + if vm, err := findVMByMoID(vmCtx, finder, uniqueID); err == nil { + return vm, nil + } + } + + // For when we start to use the k8s VM.UID for the VC VM's InstanceUUID or UUID (aka BiosUUID): + /* + if instanceUUID := vmCtx.VM.UID; instanceUUID != "" { + if vm, err := findVMByUUID(vmCtx, vimClient, datacenter, string(instanceUUID), true); err == nil { + return vm, nil + } + } + */ + + return findVMByInventory(vmCtx, k8sClient, vimClient, finder) +} + +func findVMByMoID( + vmCtx context.VirtualMachineContextA2, + finder *find.Finder, + moID string) (*object.VirtualMachine, error) { + + ref, err := finder.ObjectReference(vmCtx, types.ManagedObjectReference{Type: "VirtualMachine", Value: moID}) + if err != nil { + return nil, err + } + + vm, ok := ref.(*object.VirtualMachine) + if !ok { + return nil, fmt.Errorf("found VM reference was not a VM but a %T", ref) + } + + vmCtx.Logger.V(4).Info("Found VM via MoID", "path", vm.InventoryPath, "moID", moID) + return vm, nil +} + +//nolint:unused +func findVMByUUID( + vmCtx context.VirtualMachineContextA2, + vimClient *vim25.Client, + datacenter *object.Datacenter, + uuid string, + isInstanceUUID bool) (*object.VirtualMachine, error) { + + ref, err := object.NewSearchIndex(vimClient).FindByUuid(vmCtx, datacenter, uuid, true, &isInstanceUUID) + if err != nil { + return nil, fmt.Errorf("error finding object by UUID %q: %w", uuid, err) + } else if ref == nil { + return nil, fmt.Errorf("no VM found for UUID %q (instanceUUID: %v)", uuid, isInstanceUUID) + } + + vm, ok := ref.(*object.VirtualMachine) + if !ok { + return nil, fmt.Errorf("found VM reference was not a VirtualMachine but a %T", ref) + } + + vmCtx.Logger.V(4).Info("Found VM via UUID", "uuid", uuid, "isInstanceUUID", isInstanceUUID) + return vm, nil +} + +func findVMByInventory( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client, + vimClient *vim25.Client, + finder *find.Finder) (*object.VirtualMachine, error) { + + // Note that we'll usually only get here to find the VM via its inventory path when we're first + // creating the VM. To determine the path, we need the NS Folder MoID and the VM's ResourcePolicy, + // if set, and we'll fetch these again as a part of createVirtualMachine(). For now, just re-fetch + // but we could pass the Folder MoID and ResourcePolicy to save a bit of duplicated work. + + folderMoID, err := topology.GetNamespaceFolderMoID(vmCtx, k8sClient, vmCtx.VM.Namespace) + if err != nil { + return nil, err + } + + // While we strictly only need the Folder's ManagedObjectReference below, use the Finder + // here to check if it actually exists. + folder, err := GetFolderByMoID(vmCtx, finder, folderMoID) + if err != nil { + return nil, fmt.Errorf("failed to get namespace Folder: %w", err) + } + + // When the VM has a ResourcePolicy, the VM is placed in a child folder under the namespace's folder. + if policyName := vmCtx.VM.Spec.Reserved.ResourcePolicyName; policyName != "" { + resourcePolicy := &vmopv1.VirtualMachineSetResourcePolicy{} + + key := ctrlclient.ObjectKey{Name: policyName, Namespace: vmCtx.VM.Namespace} + if err := k8sClient.Get(vmCtx, key, resourcePolicy); err != nil { + // Note that if VM does not exist, and we're about to create it, the ResourcePolicy is Get() + // again so the corresponding condition is almost always true if we don't hit an error here. + // Creating the VM with an explicit InstanceUUID is the easiest way out to avoid this. + return nil, fmt.Errorf("failed to get VirtualMachineSetResourcePolicy: %w", err) + } + + if folderName := resourcePolicy.Spec.Folder; folderName != "" { + childFolder, err := GetChildFolder(vmCtx, folder, folderName) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get VirtualMachineSetResourcePolicy child Folder", + "parentPath", folder.InventoryPath, "folderName", folderName, "policyName", policyName) + return nil, err + } + + folder = childFolder + } + } + + ref, err := object.NewSearchIndex(vimClient).FindChild(vmCtx, folder.Reference(), vmCtx.VM.Name) + if err != nil { + return nil, err + } else if ref == nil { + // VM does not exist. + return nil, nil + } + + vm, ok := ref.(*object.VirtualMachine) + if !ok { + return nil, fmt.Errorf("found VM reference was not a VM but a %T", ref) + } + + vmCtx.Logger.V(4).Info("Found VM via inventory", + "parentFolderMoID", folder.Reference().Value, "moID", vm.Reference().Value) + return vm, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go new file mode 100644 index 000000000..29b48be5a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/getvm_test.go @@ -0,0 +1,158 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/mo" + vimtypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/apimachinery/pkg/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func getVMTests() { + Describe("GetVirtualMachine", getVM) +} + +func getVM() { + // Use a VM that vcsim creates for us. + const vcVMName = "DC0_C0_RP0_VM0" + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + + vm := builder.DummyVirtualMachineA2() + vm.Name = "getvm-test" + vm.Namespace = nsInfo.Namespace + + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + }) + + Context("Gets VM by inventory", func() { + BeforeEach(func() { + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + + task, err := vm.Clone(ctx, nsInfo.Folder, vmCtx.VM.Name, vimtypes.VirtualMachineCloneSpec{}) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + + It("returns nil if VM does not exist", func() { + vmCtx.VM.Name = "bogus" + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).To(BeNil()) + }) + + Context("Namespace Folder does not exist", func() { + BeforeEach(func() { + task, err := nsInfo.Folder.Destroy(vmCtx) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(vmCtx)).To(Succeed()) + }) + + It("returns error", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("failed to get namespace Folder")) + Expect(vm).To(BeNil()) + }) + }) + + It("returns success when MoID is invalid", func() { + // Expect fallback to inventory. + vmCtx.VM.Status.UniqueID = "vm-bogus" + + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + }) + + Context("Gets VM when MoID is set", func() { + BeforeEach(func() { + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + vmCtx.VM.Status.UniqueID = vm.Reference().Value + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + Expect(vm.Reference().Value).To(Equal(vmCtx.VM.Status.UniqueID)) + }) + }) + + // Not until we start setting either the InstanceUUID or BiosUUID + XContext("Gets VM by UUID", func() { + BeforeEach(func() { + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vm.Properties(ctx, vm.Reference(), nil, &o)).To(Succeed()) + vmCtx.VM.UID = types.UID(o.Config.InstanceUuid) + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + }) + + Context("Gets VM with ResourcePolicy by inventory", func() { + BeforeEach(func() { + resourcePolicy, folder := ctx.CreateVirtualMachineSetResourcePolicyA2("getvm-test", nsInfo) + vmCtx.VM.Spec.Reserved.ResourcePolicyName = resourcePolicy.Name + + vm, err := ctx.Finder.VirtualMachine(ctx, vcVMName) + Expect(err).ToNot(HaveOccurred()) + + task, err := vm.Clone(ctx, folder, vmCtx.VM.Name, vimtypes.VirtualMachineCloneSpec{}) + Expect(err).ToNot(HaveOccurred()) + Expect(task.Wait(ctx)).To(Succeed()) + }) + + It("returns success", func() { + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).ToNot(HaveOccurred()) + Expect(vm).ToNot(BeNil()) + }) + + It("returns error when ResourcePolicy does not exist", func() { + vmCtx.VM.Spec.Reserved.ResourcePolicyName = "bogus" + + vm, err := vcenter.GetVirtualMachine(vmCtx, ctx.Client, ctx.VCClient.Client, ctx.Datacenter, ctx.Finder) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(HavePrefix("failed to get VirtualMachineSetResourcePolicy")) + Expect(vm).To(BeNil()) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/host.go b/pkg/vmprovider/providers/vsphere2/vcenter/host.go new file mode 100644 index 000000000..e121a5a45 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/host.go @@ -0,0 +1,41 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + "context" + "fmt" + "strings" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" +) + +// GetESXHostFQDN returns the ESX host's FQDN. +func GetESXHostFQDN( + ctx context.Context, + vimClient *vim25.Client, + hostMoID string) (string, error) { + + hostMoRef := types.ManagedObjectReference{Type: "HostSystem", Value: hostMoID} + networkSys, err := object.NewHostSystem(vimClient, hostMoRef).ConfigManager().NetworkSystem(ctx) + if err != nil { + return "", fmt.Errorf("failed to get HostNetworkSystem for hostMoID %s: %w", hostMoID, err) + } + + var hostNetworkSys mo.HostNetworkSystem + if err := networkSys.Properties(ctx, networkSys.Reference(), []string{"dnsConfig"}, &hostNetworkSys); err != nil { + return "", fmt.Errorf("failed to get HostMoID %s DNSConfig prop: %w", hostMoID, err) + } + + if hostNetworkSys.DnsConfig == nil { + return "", fmt.Errorf("hostMoID %s HostNetworkSystem does not have DNSConfig", hostMoID) + } + + hostDNSConfig := hostNetworkSys.DnsConfig.GetHostDnsConfig() + hostFQDN := strings.TrimSuffix(hostDNSConfig.HostName+"."+hostDNSConfig.DomainName, ".") + return strings.ToLower(hostFQDN), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/host_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/host_test.go new file mode 100644 index 000000000..9b7fbabdc --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/host_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func hostTests() { + Describe("GetESXHostFQDN", hostFQDN) +} + +func hostFQDN() { + var ( + ctx *builder.TestContextForVCSim + testConfig builder.VCSimTestConfig + + hostMoID string + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + testConfig.WithInstanceStorage = true + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + + hosts, err := ctx.Finder.HostSystemList(ctx, "*") + Expect(err).ToNot(HaveOccurred()) + Expect(hosts).ToNot(BeEmpty()) + hostMoID = hosts[0].Reference().Value + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Describe("GetESXHostFQDN", func() { + When("host does not have DNSConfig", func() { + BeforeEach(func() { + testConfig.WithInstanceStorage = false + }) + + It("returns expected error", func() { + _, err := vcenter.GetESXHostFQDN(ctx, ctx.VCClient.Client, hostMoID) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(" does not have DNSConfig")) + }) + }) + + It("returns expected host name for host", func() { + hostName, err := vcenter.GetESXHostFQDN(ctx, ctx.VCClient.Client, hostMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(hostName).Should(Equal(fmt.Sprintf("%s.vmop.vmware.com", hostMoID))) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go new file mode 100644 index 000000000..080f1c5af --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool.go @@ -0,0 +1,163 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter + +import ( + goctx "context" + "fmt" + + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +// GetResourcePoolByMoID returns the ResourcePool for the MoID. +func GetResourcePoolByMoID( + ctx goctx.Context, + finder *find.Finder, + rpMoID string) (*object.ResourcePool, error) { + + o, err := finder.ObjectReference(ctx, types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + if err != nil { + return nil, err + } + + return o.(*object.ResourcePool), nil +} + +// GetResourcePoolOwnerMoRef returns the ClusterComputeResource MoID that owns the ResourcePool. +func GetResourcePoolOwnerMoRef( + ctx goctx.Context, + vimClient *vim25.Client, + rpMoID string) (types.ManagedObjectReference, error) { + + rp := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + + objRef, err := rp.Owner(ctx) + if err != nil { + return types.ManagedObjectReference{}, err + } + + return objRef.Reference(), nil +} + +// GetChildResourcePool gets the named child ResourcePool from the parent ResourcePool. +func GetChildResourcePool( + ctx goctx.Context, + parentRP *object.ResourcePool, + childName string) (*object.ResourcePool, error) { + + childRP, err := findChildRP(ctx, parentRP, childName) + if err != nil { + return nil, err + } else if childRP == nil { + return nil, fmt.Errorf("ResourcePool child %q not found under parent ResourcePool %s", + childName, parentRP.Reference().Value) + } + + return childRP, nil +} + +// DoesChildResourcePoolExist returns if the named child ResourcePool exists under the parent ResourcePool. +func DoesChildResourcePoolExist( + ctx goctx.Context, + vimClient *vim25.Client, + parentRPMoID, childName string) (bool, error) { + + parentRP := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: parentRPMoID}) + + childRP, err := findChildRP(ctx, parentRP, childName) + if err != nil { + return false, err + } + + return childRP != nil, nil +} + +// CreateOrUpdateChildResourcePool creates or updates the child ResourcePool under the parent ResourcePool. +func CreateOrUpdateChildResourcePool( + ctx goctx.Context, + vimClient *vim25.Client, + parentRPMoID string, + rpSpec *vmopv1.ResourcePoolSpec) (string, error) { + + parentRP := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: parentRPMoID}) + + childRP, err := findChildRP(ctx, parentRP, rpSpec.Name) + if err != nil { + return "", err + } + + spec := types.DefaultResourceConfigSpec() // TODO Set reservations & limits from rpSpec + + if childRP == nil { + rp, err := parentRP.Create(ctx, rpSpec.Name, spec) + if err != nil { + return "", err + } + + childRP = rp + } else { //nolint + // TODO: // Finish this clause + } + + return childRP.Reference().Value, nil +} + +// DeleteChildResourcePool deletes the child ResourcePool under the parent ResourcePool. +func DeleteChildResourcePool( + ctx goctx.Context, + vimClient *vim25.Client, + parentRPMoID, childName string) error { + + parentRP := object.NewResourcePool(vimClient, + types.ManagedObjectReference{Type: "ResourcePool", Value: parentRPMoID}) + + childRP, err := findChildRP(ctx, parentRP, childName) + if err != nil || childRP == nil { + return err + } + + task, err := childRP.Destroy(ctx) + if err != nil { + return err + } + + if taskResult, err := task.WaitForResult(ctx); err != nil { + if taskResult == nil || taskResult.Error == nil { + return err + } + return fmt.Errorf("destroy ResourcePool %s task failed: %w: %s", + childRP.Reference().Value, err, taskResult.Error.LocalizedMessage) + } + + return nil +} + +func findChildRP( + ctx goctx.Context, + parentRP *object.ResourcePool, + childName string) (*object.ResourcePool, error) { + + objRef, err := object.NewSearchIndex(parentRP.Client()).FindChild(ctx, parentRP, childName) + if err != nil { + return nil, err + } else if objRef == nil { + // FindChild() returns nil when child name is not found. + return nil, nil + } + + childRP, ok := objRef.(*object.ResourcePool) + if !ok { + return nil, fmt.Errorf("ResourcePool child %q is not a ResourcePool but a %T", childName, objRef) + } + + return childRP, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go new file mode 100644 index 000000000..e0e16208d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/resourcepool_test.go @@ -0,0 +1,200 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func resourcePoolTests() { + Describe("GetResourcePool", getResourcePoolTests) + Describe("CreateDeleteExistResourcePoolChild", createDeleteExistResourcePoolChild) +} + +func getResourcePoolTests() { + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + nsRP *object.ResourcePool + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + nsRP = ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + nsRP = nil + }) + + Context("GetResourcePoolByMoID", func() { + It("returns success", func() { + rp, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, nsRP.Reference().Value) + Expect(err).ToNot(HaveOccurred()) + Expect(rp).ToNot(BeNil()) + Expect(rp.Reference()).To(Equal(nsRP.Reference())) + }) + + It("returns error when MoID does not exist", func() { + rp, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, "bogus") + Expect(err).To(HaveOccurred()) + Expect(rp).To(BeNil()) + }) + }) + + Context("GetResourcePoolOwnerMoRef", func() { + It("returns success", func() { + ccr, err := vcenter.GetResourcePoolOwnerMoRef(ctx, ctx.VCClient.Client, nsRP.Reference().Value) + Expect(err).ToNot(HaveOccurred()) + Expect(ccr).To(Equal(ctx.GetSingleClusterCompute().Reference())) + }) + + It("returns error when MoID does not exist", func() { + _, err := vcenter.GetResourcePoolOwnerMoRef(ctx, ctx.VCClient.Client, "bogus") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("GetChildResourcePool", func() { + It("returns success", func() { + // Quick way for a child RP is to create a VMSetResourcePolicy. + resourcePolicy, _ := ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + childRPName := resourcePolicy.Spec.ResourcePool.Name + Expect(childRPName).ToNot(BeEmpty()) + + childRP, err := vcenter.GetChildResourcePool(ctx, nsRP, childRPName) + Expect(err).ToNot(HaveOccurred()) + Expect(childRP).ToNot(BeNil()) + + objRef, err := ctx.Finder.ObjectReference(ctx, childRP.Reference()) + Expect(err).ToNot(HaveOccurred()) + childRP, ok := objRef.(*object.ResourcePool) + Expect(ok).To(BeTrue()) + Expect(childRP.Name()).To(Equal(resourcePolicy.Name)) + }) + + It("returns error when child RP does not exist", func() { + childRP, err := vcenter.GetChildResourcePool(ctx, nsRP, "bogus") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not found under parent ResourcePool")) + Expect(childRP).To(BeNil()) + }) + }) +} + +func createDeleteExistResourcePoolChild() { + + var ( + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + nsRP *object.ResourcePool + + parentRPMoID string + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + nsInfo = ctx.CreateWorkloadNamespace() + nsRP = ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + + parentRPMoID = nsRP.Reference().Value + + resourcePolicy, _ = ctx.CreateVirtualMachineSetResourcePolicyA2("my-child-rp", nsInfo) + Expect(resourcePolicy).ToNot(BeNil()) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + nsRP = nil + parentRPMoID = "" + resourcePolicy = nil + }) + + Context("CreateOrUpdateChildResourcePool", func() { + It("creates child ResourcePool", func() { + childMoID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + Expect(childMoID).ToNot(BeEmpty()) + + By("returns success when child ResourcePool already exists", func() { + moID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + Expect(moID).To(Equal(childMoID)) + }) + + By("child ResourcePool is found by MoID", func() { + rp, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, childMoID) + Expect(err).ToNot(HaveOccurred()) + Expect(rp.Reference().Value).To(Equal(childMoID)) + }) + }) + + It("returns error when when parent ResourcePool MoID does not exist", func() { + childMoID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, "bogus", &resourcePolicy.Spec.ResourcePool) + Expect(err).To(HaveOccurred()) + Expect(childMoID).To(BeEmpty()) + }) + }) + + Context("DoesChildResourcePoolExist", func() { + It("returns true when child ResourcePool exists", func() { + childName := resourcePolicy.Spec.ResourcePool.Name + + _, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + + exists, err := vcenter.DoesChildResourcePoolExist(ctx, ctx.VCClient.Client, parentRPMoID, childName) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + + It("returns false when child ResourcePool does not exist", func() { + exists, err := vcenter.DoesChildResourcePoolExist(ctx, ctx.VCClient.Client, parentRPMoID, "bogus") + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + + It("returns error when parent ResourcePool MoID does not exist", func() { + exists, err := vcenter.DoesChildResourcePoolExist(ctx, ctx.VCClient.Client, "bogus", "bogus") + Expect(err).To(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("DeleteChildResourcePool", func() { + It("deletes child ResourcePool", func() { + childName := resourcePolicy.Spec.ResourcePool.Name + + childMoID, err := vcenter.CreateOrUpdateChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, &resourcePolicy.Spec.ResourcePool) + Expect(err).ToNot(HaveOccurred()) + + err = vcenter.DeleteChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, childName) + Expect(err).ToNot(HaveOccurred()) + + By("child ResourcePool is not found by MoID", func() { + _, err := vcenter.GetResourcePoolByMoID(ctx, ctx.Finder, childMoID) + Expect(err).To(HaveOccurred()) + }) + + By("NoOp when child does not exist", func() { + err := vcenter.DeleteChildResourcePool(ctx, ctx.VCClient.Client, parentRPMoID, childName) + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go b/pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go new file mode 100644 index 000000000..63d223192 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vcenter/vcenter_suite_test.go @@ -0,0 +1,30 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vcenter_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() + +func vcSimTests() { + Describe("Cluster", clusterTests) + Describe("Folder", folderTests) + Describe("GetVM", getVMTests) + Describe("Host", hostTests) + Describe("ResourcePool", resourcePoolTests) +} + +func TestVCenter(t *testing.T) { + suite.Register(t, "VMProvider VCenter Tests", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go new file mode 100644 index 000000000..ff752d426 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr.go @@ -0,0 +1,34 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + "fmt" + + "github.com/vmware/govmomi/object" +) + +// GetVMClusterComputeResource returns the VM's ClusterComputeResource. +func GetVMClusterComputeResource( + ctx context.Context, + vcVM *object.VirtualMachine) (*object.ClusterComputeResource, error) { + + rp, err := vcVM.ResourcePool(ctx) + if err != nil { + return nil, err + } + + ccrRef, err := rp.Owner(ctx) + if err != nil { + return nil, err + } + + cluster, ok := ccrRef.(*object.ClusterComputeResource) + if !ok { + return nil, fmt.Errorf("VM Owner is not a ClusterComputeResource but %T", ccrRef) + } + + return cluster, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go new file mode 100644 index 000000000..55e4a5842 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/ccr_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func ccrTests() { + + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Returns VM ClusterComputeResource", func() { + ccr, err := virtualmachine.GetVMClusterComputeResource(ctx, vcVM) + Expect(err).ToNot(HaveOccurred()) + Expect(ccr).ToNot(BeNil()) + Expect(ccr.Reference()).To(Equal(ctx.GetSingleClusterCompute().Reference())) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go new file mode 100644 index 000000000..5ae236f88 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go @@ -0,0 +1,190 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "github.com/vmware/govmomi/vim25/types" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" +) + +// CreateConfigSpec returns an initial ConfigSpec that is created by overlaying the +// base ConfigSpec with VM Class spec and other arguments. +// TODO: We eventually need to de-dupe much of this with the ConfigSpec manipulation that's later done +// in the "update" pre-power on path. That operates on a ConfigInfo so we'd need to populate that from +// the config we build here. +func CreateConfigSpec( + vmCtx context.VirtualMachineContextA2, + vmClassConfigSpec *types.VirtualMachineConfigSpec, + vmClassSpec *vmopv1.VirtualMachineClassSpec, + vmImageStatus *vmopv1.VirtualMachineImageStatus, + minFreq uint64) *types.VirtualMachineConfigSpec { + + configSpec := types.VirtualMachineConfigSpec{} + + // If there is a class ConfigSpec, then that is our initial ConfigSpec. + if vmClassConfigSpec != nil { + configSpec = *vmClassConfigSpec + } + + configSpec.Name = vmCtx.VM.Name + if configSpec.Annotation == "" { + // If the class ConfigSpec doesn't specify any annotations, set the default one. + configSpec.Annotation = constants.VCVMAnnotation + } + // CPU and Memory configurations specified in the VM Class standalone fields take + // precedence over values in the config spec + configSpec.NumCPUs = int32(vmClassSpec.Hardware.Cpus) + configSpec.MemoryMB = MemoryQuantityToMb(vmClassSpec.Hardware.Memory) + configSpec.ManagedBy = &types.ManagedByInfo{ + ExtensionKey: constants.ManagedByExtensionKey, + Type: constants.ManagedByExtensionType, + } + + if val, ok := vmCtx.VM.Annotations[constants.FirmwareOverrideAnnotation]; ok { + configSpec.Firmware = val + } else if configSpec.Firmware == "" && vmImageStatus != nil { + // Use firmware type from the image if ConfigSpec doesn't have it. + configSpec.Firmware = vmImageStatus.Firmware + } + + // TODO: Otherwise leave as-is? Our ChangeBlockTracking could be better as a *bool. + if vmCtx.VM.Spec.Advanced.ChangeBlockTracking { + configSpec.ChangeTrackingEnabled = pointer.Bool(true) + } + + // Populate the CPU reservation and limits in the ConfigSpec if VAPI fields specify any. + // VM Class VAPI does not support Limits, so they will never be non nil. + // TODO: Remove limits: issues/56 + if res := vmClassSpec.Policies.Resources; !res.Requests.Cpu.IsZero() || !res.Limits.Cpu.IsZero() { + // TODO: Always override? + configSpec.CpuAllocation = &types.ResourceAllocationInfo{ + Shares: &types.SharesInfo{ + Level: types.SharesLevelNormal, + }, + } + + if !res.Requests.Cpu.IsZero() { + rsv := CPUQuantityToMhz(vmClassSpec.Policies.Resources.Requests.Cpu, minFreq) + configSpec.CpuAllocation.Reservation = &rsv + } + if !res.Limits.Cpu.IsZero() { + lim := CPUQuantityToMhz(vmClassSpec.Policies.Resources.Limits.Cpu, minFreq) + configSpec.CpuAllocation.Limit = &lim + } + } + + // Populate the memory reservation and limits in the ConfigSpec if VAPI fields specify any. + // TODO: Remove limits: issues/56 + if res := vmClassSpec.Policies.Resources; !res.Requests.Memory.IsZero() || !res.Limits.Memory.IsZero() { + // TODO: Always override? + configSpec.MemoryAllocation = &types.ResourceAllocationInfo{ + Shares: &types.SharesInfo{ + Level: types.SharesLevelNormal, + }, + } + + if !res.Requests.Memory.IsZero() { + rsv := MemoryQuantityToMb(vmClassSpec.Policies.Resources.Requests.Memory) + configSpec.MemoryAllocation.Reservation = &rsv + } + if !res.Limits.Memory.IsZero() { + lim := MemoryQuantityToMb(vmClassSpec.Policies.Resources.Limits.Memory) + configSpec.MemoryAllocation.Limit = &lim + } + } + + return &configSpec +} + +// CreateConfigSpecForPlacement creates a ConfigSpec that is suitable for Placement. +// baseConfigSpec will likely be - or at least derived from - the ConfigSpec returned by CreateConfigSpec above. +func CreateConfigSpecForPlacement( + vmCtx context.VirtualMachineContextA2, + baseConfigSpec *types.VirtualMachineConfigSpec, + storageClassesToIDs map[string]string) *types.VirtualMachineConfigSpec { + + // TODO: If placement chokes on EthCards w/o a backing yet (NSX-T) remove those entries here. + deviceChangeCopy := make([]types.BaseVirtualDeviceConfigSpec, len(baseConfigSpec.DeviceChange)) + copy(deviceChangeCopy, baseConfigSpec.DeviceChange) + + configSpec := *baseConfigSpec + configSpec.DeviceChange = deviceChangeCopy + + // Add a dummy disk for placement: PlaceVmsXCluster expects there to always be at least one disk. + // Until we're in a position to have the OVF envelope here, add a dummy disk satisfy it. + configSpec.DeviceChange = append(configSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate, + Device: &types.VirtualDisk{ + CapacityInBytes: 1024 * 1024, + VirtualDevice: types.VirtualDevice{ + Key: -42, + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(true), + }, + }, + }, + Profile: []types.BaseVirtualMachineProfileSpec{ + &types.VirtualMachineDefinedProfileSpec{ + ProfileId: storageClassesToIDs[vmCtx.VM.Spec.StorageClass], + }, + }, + }) + + if lib.IsInstanceStorageFSSEnabled() { + isVolumes := instancestorage.FilterVolumes(vmCtx.VM) + + for idx, dev := range CreateInstanceStorageDiskDevices(isVolumes) { + configSpec.DeviceChange = append(configSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate, + Device: dev, + Profile: []types.BaseVirtualMachineProfileSpec{ + &types.VirtualMachineDefinedProfileSpec{ + ProfileId: storageClassesToIDs[isVolumes[idx].PersistentVolumeClaim.InstanceVolumeClaim.StorageClass], + ProfileData: &types.VirtualMachineProfileRawData{ + ExtensionKey: "com.vmware.vim.sps", + }, + }, + }, + }) + } + } + + // TODO: Add more devices and fields + // - boot disks from OVA + // - storage profile/class + // - PVC volumes + // - Network devices (meh for now b/c of wcp constraints) + // - anything in ExtraConfig matter here? + // - any way to do the cluster modules for anti-affinity? + // - whatever else I'm forgetting + + return &configSpec +} + +// ConfigSpecFromVMClassDevices creates a ConfigSpec that adds the standalone hardware devices from +// the VMClass if any. This ConfigSpec will be used as the class ConfigSpec to CreateConfigSpec, with +// the rest of the class fields - like CPU count - applied on top. +func ConfigSpecFromVMClassDevices(vmClassSpec *vmopv1.VirtualMachineClassSpec) *types.VirtualMachineConfigSpec { + devsFromClass := CreatePCIDevicesFromVMClass(vmClassSpec.Hardware.Devices) + if len(devsFromClass) == 0 { + return nil + } + + configSpec := &types.VirtualMachineConfigSpec{} + for _, dev := range devsFromClass { + configSpec.DeviceChange = append(configSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: dev, + }) + } + return configSpec +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go new file mode 100644 index 000000000..0dd01e72c --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go @@ -0,0 +1,278 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + goctx "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + vimtypes "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("CreateConfigSpec", func() { + const vmName = "dummy-vm" + + var ( + vmCtx context.VirtualMachineContextA2 + vmClassSpec *vmopv1.VirtualMachineClassSpec + vmImageStatus *vmopv1.VirtualMachineImageStatus + minCPUFreq uint64 + configSpec *vimtypes.VirtualMachineConfigSpec + classConfigSpec *vimtypes.VirtualMachineConfigSpec + err error + ) + + BeforeEach(func() { + vmClass := builder.DummyVirtualMachineClassA2() + vmClassSpec = &vmClass.Spec + vmImageStatus = &vmopv1.VirtualMachineImageStatus{Firmware: "efi"} + minCPUFreq = 2500 + + vm := builder.DummyVirtualMachineA2() + vm.Name = vmName + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithValues("vmName", vm.GetName()), + VM: vm, + } + }) + + It("Basic ConfigSpec assertions", func() { + configSpec = virtualmachine.CreateConfigSpec( + vmCtx, + nil, + vmClassSpec, + vmImageStatus, + minCPUFreq) + + Expect(configSpec).ToNot(BeNil()) + Expect(err).To(BeNil()) + Expect(configSpec.Name).To(Equal(vmName)) + Expect(configSpec.Annotation).ToNot(BeEmpty()) + Expect(configSpec.NumCPUs).To(BeEquivalentTo(vmClassSpec.Hardware.Cpus)) + Expect(configSpec.MemoryMB).To(BeEquivalentTo(4 * 1024)) + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.Firmware).To(Equal(vmImageStatus.Firmware)) + }) + + Context("Use VM Class ConfigSpec", func() { + BeforeEach(func() { + classConfigSpec = &vimtypes.VirtualMachineConfigSpec{ + Name: "dont-use-this-dummy-VM", + Annotation: "test-annotation", + DeviceChange: []vimtypes.BaseVirtualDeviceConfigSpec{ + &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimtypes.VirtualE1000{ + VirtualEthernetCard: vimtypes.VirtualEthernetCard{ + VirtualDevice: vimtypes.VirtualDevice{ + Key: 4000, + }, + }, + }, + }, + }, + } + }) + + JustBeforeEach(func() { + configSpec = virtualmachine.CreateConfigSpec( + vmCtx, + classConfigSpec, + vmClassSpec, + vmImageStatus, + minCPUFreq) + Expect(configSpec).ToNot(BeNil()) + }) + + It("Returns expected config spec", func() { + Expect(configSpec.Name).To(Equal(vmName)) + Expect(configSpec.Annotation).ToNot(BeEmpty()) + Expect(configSpec.Annotation).To(Equal("test-annotation")) + Expect(configSpec.NumCPUs).To(BeEquivalentTo(vmClassSpec.Hardware.Cpus)) + Expect(configSpec.MemoryMB).To(BeEquivalentTo(4 * 1024)) + Expect(configSpec.CpuAllocation).ToNot(BeNil()) + Expect(configSpec.MemoryAllocation).ToNot(BeNil()) + Expect(configSpec.Firmware).To(Equal(vmImageStatus.Firmware)) + Expect(configSpec.DeviceChange).To(HaveLen(1)) + dSpec := configSpec.DeviceChange[0].GetVirtualDeviceConfigSpec() + _, ok := dSpec.Device.(*vimtypes.VirtualE1000) + Expect(ok).To(BeTrue()) + }) + }) +}) + +var _ = Describe("CreateConfigSpecForPlacement", func() { + + var ( + vmCtx context.VirtualMachineContextA2 + storageClassesToIDs map[string]string + baseConfigSpec *vimtypes.VirtualMachineConfigSpec + configSpec *vimtypes.VirtualMachineConfigSpec + ) + + BeforeEach(func() { + baseConfigSpec = &vimtypes.VirtualMachineConfigSpec{} + storageClassesToIDs = map[string]string{} + + vm := builder.DummyVirtualMachineA2() + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithValues("vmName", vm.GetName()), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec = virtualmachine.CreateConfigSpecForPlacement( + vmCtx, + baseConfigSpec, + storageClassesToIDs) + Expect(configSpec).ToNot(BeNil()) + }) + + Context("Returns expected ConfigSpec", func() { + BeforeEach(func() { + baseConfigSpec = &vimtypes.VirtualMachineConfigSpec{ + Name: "dummy-VM", + Annotation: "test-annotation", + NumCPUs: 42, + MemoryMB: 4096, + Firmware: "secret-sauce", + DeviceChange: []vimtypes.BaseVirtualDeviceConfigSpec{ + &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationAdd, + Device: &vimtypes.VirtualPCIPassthrough{ + VirtualDevice: vimtypes.VirtualDevice{ + Backing: &vimtypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "SampleProfile2", + }, + }, + }, + }, + }, + } + }) + + It("Placement ConfigSpec contains expected field set sans ethernet device from class config spec", func() { + Expect(configSpec.Annotation).ToNot(BeEmpty()) + Expect(configSpec.Annotation).To(Equal(baseConfigSpec.Annotation)) + Expect(configSpec.NumCPUs).To(Equal(baseConfigSpec.NumCPUs)) + Expect(configSpec.MemoryMB).To(Equal(baseConfigSpec.MemoryMB)) + Expect(configSpec.CpuAllocation).To(Equal(baseConfigSpec.CpuAllocation)) + Expect(configSpec.MemoryAllocation).To(Equal(baseConfigSpec.MemoryAllocation)) + Expect(configSpec.Firmware).To(Equal(baseConfigSpec.Firmware)) + + Expect(configSpec.DeviceChange).To(HaveLen(2)) + dSpec := configSpec.DeviceChange[0].GetVirtualDeviceConfigSpec() + _, ok := dSpec.Device.(*vimtypes.VirtualPCIPassthrough) + Expect(ok).To(BeTrue()) + dSpec1 := configSpec.DeviceChange[1].GetVirtualDeviceConfigSpec() + _, ok = dSpec1.Device.(*vimtypes.VirtualDisk) + Expect(ok).To(BeTrue()) + }) + }) + + Context("When InstanceStorage is configured", func() { + const storagePolicyID = "storage-id-42" + var oldIsInstanceStorageFSSEnabled func() bool + + BeforeEach(func() { + oldIsInstanceStorageFSSEnabled = lib.IsInstanceStorageFSSEnabled + lib.IsInstanceStorageFSSEnabled = func() bool { return true } + + builder.AddDummyInstanceStorageVolumeA2(vmCtx.VM) + storageClassesToIDs[builder.DummyStorageClassName] = storagePolicyID + }) + + AfterEach(func() { + lib.IsInstanceStorageFSSEnabled = oldIsInstanceStorageFSSEnabled + }) + + It("ConfigSpec contains expected InstanceStorage devices", func() { + Expect(configSpec.DeviceChange).To(HaveLen(3)) + assertInstanceStorageDeviceChange(configSpec.DeviceChange[1], 256, storagePolicyID) + assertInstanceStorageDeviceChange(configSpec.DeviceChange[2], 512, storagePolicyID) + }) + }) +}) + +var _ = Describe("ConfigSpecFromVMClassDevices", func() { + + var ( + vmClassSpec *vmopv1.VirtualMachineClassSpec + configSpec *vimtypes.VirtualMachineConfigSpec + ) + + Context("when Class specifies GPU/DDPIO in Hardware", func() { + + BeforeEach(func() { + vmClassSpec = &vmopv1.VirtualMachineClassSpec{} + + vmClassSpec.Hardware.Devices.VGPUDevices = []vmopv1.VGPUDevice{{ + ProfileName: "createplacementspec-profile", + }} + + vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices = []vmopv1.DynamicDirectPathIODevice{{ + VendorID: 20, + DeviceID: 30, + CustomLabel: "createplacementspec-label", + }} + }) + + JustBeforeEach(func() { + configSpec = virtualmachine.ConfigSpecFromVMClassDevices(vmClassSpec) + }) + + It("Returns expected ConfigSpec", func() { + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.DeviceChange).To(HaveLen(2)) // One each for GPU an DDPIO above + + dSpec1 := configSpec.DeviceChange[0].GetVirtualDeviceConfigSpec() + dev1, ok := dSpec1.Device.(*vimtypes.VirtualPCIPassthrough) + Expect(ok).To(BeTrue()) + pciDev1 := dev1.GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*vimtypes.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).To(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal(vmClassSpec.Hardware.Devices.VGPUDevices[0].ProfileName)) + + dSpec2 := configSpec.DeviceChange[1].GetVirtualDeviceConfigSpec() + dev2, ok2 := dSpec2.Device.(*vimtypes.VirtualPCIPassthrough) + Expect(ok2).To(BeTrue()) + pciDev2 := dev2.GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*vimtypes.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).To(BeTrue()) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(BeEquivalentTo(vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices[0].DeviceID)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(BeEquivalentTo(vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices[0].VendorID)) + Expect(pciBacking2.CustomLabel).To(Equal(vmClassSpec.Hardware.Devices.DynamicDirectPathIODevices[0].CustomLabel)) + }) + }) +}) + +func assertInstanceStorageDeviceChange( + deviceChange vimtypes.BaseVirtualDeviceConfigSpec, + expectedSizeGB int, + expectedStoragePolicyID string) { + + dc := deviceChange.GetVirtualDeviceConfigSpec() + Expect(dc.Operation).To(Equal(vimtypes.VirtualDeviceConfigSpecOperationAdd)) + Expect(dc.FileOperation).To(Equal(vimtypes.VirtualDeviceConfigSpecFileOperationCreate)) + + dev, ok := dc.Device.(*vimtypes.VirtualDisk) + Expect(ok).To(BeTrue()) + Expect(dev.CapacityInBytes).To(BeEquivalentTo(expectedSizeGB * 1024 * 1024 * 1024)) + + Expect(dc.Profile).To(HaveLen(1)) + profile, ok := dc.Profile[0].(*vimtypes.VirtualMachineDefinedProfileSpec) + Expect(ok).To(BeTrue()) + Expect(profile.ProfileId).To(Equal(expectedStoragePolicyID)) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go new file mode 100644 index 000000000..06325c5e6 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion.go @@ -0,0 +1,18 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "math" + + "k8s.io/apimachinery/pkg/api/resource" +) + +func MemoryQuantityToMb(q resource.Quantity) int64 { + return int64(math.Ceil(float64(q.Value()) / float64(1024*1024))) +} + +func CPUQuantityToMhz(q resource.Quantity, cpuFreqMhz uint64) int64 { + return int64(math.Ceil(float64(q.MilliValue()) * float64(cpuFreqMhz) / float64(1000))) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go new file mode 100644 index 000000000..cb53717eb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/conversion_test.go @@ -0,0 +1,34 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" +) + +var _ = Describe("CPUQuantityToMhz", func() { + + Context("Convert CPU units from milli-cores to MHz", func() { + It("return whole number for non-integer CPU quantity", func() { + q, err := resource.ParseQuantity("500m") + Expect(err).NotTo(HaveOccurred()) + freq := virtualmachine.CPUQuantityToMhz(q, 3225) + expectVal := int64(1613) + Expect(freq).Should(BeNumerically("==", expectVal)) + }) + + It("return whole number for integer CPU quantity", func() { + q, err := resource.ParseQuantity("1000m") + Expect(err).NotTo(HaveOccurred()) + freq := virtualmachine.CPUQuantityToMhz(q, 3225) + expectVal := int64(3225) + Expect(freq).Should(BeNumerically("==", expectVal)) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go new file mode 100644 index 000000000..7641eaacb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete.go @@ -0,0 +1,44 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "github.com/go-logr/logr" + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + vmutil "github.com/vmware-tanzu/vm-operator/pkg/util/vsphere/vm" +) + +func DeleteVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine) error { + + if _, err := vmutil.SetAndWaitOnPowerState( + logr.NewContext(vmCtx, vmCtx.Logger), + vcVM.Client(), + vmutil.ManagedObjectFromObject(vcVM), + false, + types.VirtualMachinePowerStatePoweredOff, + vmutil.ParsePowerOpMode(string(vmCtx.VM.Spec.PowerOffMode))); err != nil { + + return err + } + + t, err := vcVM.Destroy(vmCtx) + if err != nil { + return err + } + + if taskInfo, err := t.WaitForResult(vmCtx); err != nil { + if taskInfo != nil { + vmCtx.Logger.V(5).Error(err, "destroy VM task failed", "taskInfo", taskInfo) + } + return errors.Wrapf(err, "destroy VM task failed") + } + + return nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go new file mode 100644 index 000000000..3be4c39f5 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/delete_test.go @@ -0,0 +1,71 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func deleteTests() { + + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vcVM.Name()), + VM: builder.DummyVirtualMachineA2(), + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Deletes VM that is off", func() { + moID := vcVM.Reference().Value + Expect(ctx.GetVMFromMoID(moID)).ToNot(BeNil()) + + t, err := vcVM.PowerOff(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Wait(ctx)).To(Succeed()) + + err = virtualmachine.DeleteVirtualMachine(vmCtx, vcVM) + Expect(err).ToNot(HaveOccurred()) + + Expect(ctx.GetVMFromMoID(moID)).To(BeNil()) + }) + + It("Deletes VM that is on", func() { + moID := vcVM.Reference().Value + Expect(ctx.GetVMFromMoID(moID)).ToNot(BeNil()) + + state, err := vcVM.PowerState(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + + err = virtualmachine.DeleteVirtualMachine(vmCtx, vcVM) + Expect(err).ToNot(HaveOccurred()) + + Expect(ctx.GetVMFromMoID(moID)).To(BeNil()) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go new file mode 100644 index 000000000..1152ea435 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/devices.go @@ -0,0 +1,100 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + vimTypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +const ( + // A negative device range is traditionally used. + pciDevicesStartDeviceKey = int32(-200) + instanceStorageStartDeviceKey = int32(-300) +) + +func CreatePCIPassThroughDevice(deviceKey int32, backingInfo vimTypes.BaseVirtualDeviceBackingInfo) vimTypes.BaseVirtualDevice { + device := &vimTypes.VirtualPCIPassthrough{ + VirtualDevice: vimTypes.VirtualDevice{ + Key: deviceKey, + Backing: backingInfo, + }, + } + return device +} + +// CreatePCIDevicesFromConfigSpec creates vim25 VirtualDevices from the specified list of PCI devices from the VM Class ConfigSpec. +func CreatePCIDevicesFromConfigSpec(pciDevsFromConfigSpec []*vimTypes.VirtualPCIPassthrough) []vimTypes.BaseVirtualDevice { + devices := make([]vimTypes.BaseVirtualDevice, 0, len(pciDevsFromConfigSpec)) + + deviceKey := pciDevicesStartDeviceKey + + for i := range pciDevsFromConfigSpec { + dev := pciDevsFromConfigSpec[i] + dev.Key = deviceKey + devices = append(devices, dev) + deviceKey-- + } + + return devices +} + +// CreatePCIDevicesFromVMClass creates vim25 VirtualDevices from the specified list of PCI devices from VM Class spec. +func CreatePCIDevicesFromVMClass(pciDevicesFromVMClass vmopv1.VirtualDevices) []vimTypes.BaseVirtualDevice { + devices := make([]vimTypes.BaseVirtualDevice, 0, len(pciDevicesFromVMClass.VGPUDevices)+len(pciDevicesFromVMClass.DynamicDirectPathIODevices)) + + deviceKey := pciDevicesStartDeviceKey + + for _, vGPU := range pciDevicesFromVMClass.VGPUDevices { + backingInfo := &vimTypes.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: vGPU.ProfileName, + } + dev := CreatePCIPassThroughDevice(deviceKey, backingInfo) + devices = append(devices, dev) + deviceKey-- + } + + for _, dynamicDirectPath := range pciDevicesFromVMClass.DynamicDirectPathIODevices { + allowedDev := vimTypes.VirtualPCIPassthroughAllowedDevice{ + VendorId: int32(dynamicDirectPath.VendorID), + DeviceId: int32(dynamicDirectPath.DeviceID), + } + backingInfo := &vimTypes.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []vimTypes.VirtualPCIPassthroughAllowedDevice{allowedDev}, + CustomLabel: dynamicDirectPath.CustomLabel, + } + dev := CreatePCIPassThroughDevice(deviceKey, backingInfo) + devices = append(devices, dev) + deviceKey-- + } + + return devices +} + +func CreateInstanceStorageDiskDevices(isVolumes []vmopv1.VirtualMachineVolume) []vimTypes.BaseVirtualDevice { + devices := make([]vimTypes.BaseVirtualDevice, 0, len(isVolumes)) + deviceKey := instanceStorageStartDeviceKey + + for _, volume := range isVolumes { + device := &vimTypes.VirtualDisk{ + CapacityInBytes: volume.PersistentVolumeClaim.InstanceVolumeClaim.Size.Value(), + VirtualDevice: vimTypes.VirtualDevice{ + Key: deviceKey, + Backing: &vimTypes.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(false), + }, + }, + VDiskId: &vimTypes.ID{ + Id: constants.InstanceStorageVDiskID, + }, + } + devices = append(devices, device) + deviceKey-- + } + + return devices +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go new file mode 100644 index 000000000..424f8b527 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/heartbeat.go @@ -0,0 +1,25 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "context" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +func GetGuestHeartBeatStatus( + ctx context.Context, + vm *object.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { + + var o mo.VirtualMachine + if err := vm.Properties(ctx, vm.Reference(), []string{"guestHeartbeatStatus"}, &o); err != nil { + return "", err + } + + return vmopv1.GuestHeartbeatStatus(o.GuestHeartbeatStatus), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go new file mode 100644 index 000000000..f1f13db2b --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish.go @@ -0,0 +1,64 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "fmt" + "net/http" + + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vapi/vcenter" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +const ( + sourceVirtualMachineType = "VirtualMachine" + + // vAPICtxActIDHttpHeader represents the http header in vAPI to pass down the activation ID. + vAPICtxActIDHttpHeader = "vapi-ctx-actid" + + itemDescriptionFormat = "virtualmachinepublishrequest.vmoperator.vmware.com: %s\n" +) + +func CreateOVF( + vmCtx context.VirtualMachineContextA2, + client *rest.Client, + vmPubReq *vmopv1.VirtualMachinePublishRequest, + cl *imgregv1a1.ContentLibrary, + actID string) (string, error) { + + // Use VM Operator specific description so that we can link published items + // to the vmPub if anything unexpected happened. + descriptionPrefix := fmt.Sprintf(itemDescriptionFormat, string(vmPubReq.UID)) + createSpec := vcenter.CreateSpec{ + Name: vmPubReq.Status.TargetRef.Item.Name, + Description: descriptionPrefix + vmPubReq.Status.TargetRef.Item.Description, + } + + source := vcenter.ResourceID{ + Type: sourceVirtualMachineType, + Value: vmCtx.VM.Status.UniqueID, + } + + target := vcenter.LibraryTarget{ + LibraryID: string(cl.Spec.UUID), + } + + ovf := vcenter.OVF{ + Spec: createSpec, + Source: source, + Target: target, + } + + vmCtx.Logger.Info("Creating OVF from VM", "spec", ovf, "actId", actID) + + // Use vmpublish uid as the act id passed down to the content library service, so that we can track + // the task status by the act id. + ctxHeader := client.WithHeader(vmCtx, http.Header{vAPICtxActIDHttpHeader: []string{actID}}) + return vcenter.NewManager(client).CreateOVF(ctxHeader, ovf) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go new file mode 100644 index 000000000..657f4de15 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/publish_test.go @@ -0,0 +1,98 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func publishTests() { + + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + vm *vmopv1.VirtualMachine + cl *imgregv1a1.ContentLibrary + vmPub *vmopv1.VirtualMachinePublishRequest + vmCtx context.VirtualMachineContextA2 + vmPubCtx context.VirtualMachinePublishRequestContext + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true, WithContentLibrary: true}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + + vm = builder.DummyVirtualMachineA2() + vm.Status.UniqueID = vcVM.Reference().Value + cl = builder.DummyContentLibrary("dummy-cl", "dummy-ns", ctx.ContentLibraryID) + vmPub = builder.DummyVirtualMachinePublishRequestA2("dummy-vmpub", "dummy-ns", + vcVM.Name(), "dummy-item-name", "dummy-cl") + vmPub.Status.SourceRef = &vmPub.Spec.Source + vmPub.Status.TargetRef = &vmPub.Spec.Target + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vcVM.Name()), + VM: vm, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + It("Publishes VM that is off", func() { + t, err := vcVM.PowerOff(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(t.Wait(ctx)).To(Succeed()) + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).ToNot(HaveOccurred()) + Expect(itemID).NotTo(BeNil()) + }) + + It("Publishes VM that is on", func() { + state, err := vcVM.PowerState(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).ToNot(HaveOccurred()) + Expect(itemID).NotTo(BeNil()) + }) + + // TODO: update after vcsim bug is resolved. + // Currently if cl doesn't exist, vcsim set notFound http code + // but doesn't return immediately, which cause a panic error. + XIt("returns error if target content library does not exist", func() { + vmPubCtx.ContentLibrary.Spec.UUID = "12345" + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).To(HaveOccurred()) + Expect(itemID).To(BeEmpty()) + }) + + // TODO: vcsim currently doesn't check if an item already exists in the cl. + XIt("returns error if target content library item already exists", func() { + vmPubCtx.VMPublishRequest.Spec.Target.Item.Name = ctx.ContentLibraryImageName + + itemID, err := virtualmachine.CreateOVF(vmCtx, ctx.RestClient, vmPub, cl, "") + Expect(err).ToNot(HaveOccurred()) + Expect(itemID).NotTo(BeNil()) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go new file mode 100644 index 000000000..8ba9c7376 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/storage.go @@ -0,0 +1,41 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/storage" +) + +// GetDefaultDiskProvisioningType gets the default disk provisioning type specified for the VM. +func GetDefaultDiskProvisioningType( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + storageProfileID string) (string, error) { + + switch vmCtx.VM.Spec.Advanced.DefaultVolumeProvisioningMode { + case vmopv1.VirtualMachineVolumeProvisioningModeThin: + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeThin), nil + case vmopv1.VirtualMachineVolumeProvisioningModeThick: + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeThick), nil + case vmopv1.VirtualMachineVolumeProvisioningModeThickEagerZero: + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeEagerZeroedThick), nil + } + + if storageProfileID != "" { + provisioning, err := storage.GetDiskProvisioningForProfile(vmCtx, vcClient, storageProfileID) + if err != nil { + return "", err + } + if provisioning != "" { + return provisioning, nil + } + } + + return string(types.OvfCreateImportSpecParamsDiskProvisioningTypeThin), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go new file mode 100644 index 000000000..12c5c35fd --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go @@ -0,0 +1,28 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vcSimTests() { + Describe("ClusterComputeResource", ccrTests) + Describe("Delete", deleteTests) + Describe("Publish", publishTests) +} + +var suite = builder.NewTestSuite() + +func TestClusterModules(t *testing.T) { + suite.Register(t, "vSphere Provider VirtualMachine Suite", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go new file mode 100644 index 000000000..cc311b468 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket.go @@ -0,0 +1,65 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/sha512" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" +) + +func GetWebConsoleTicket( + vmCtx context.VirtualMachineContextA2, + vm *object.VirtualMachine, + pubKey string) (string, error) { + + vmCtx.Logger.V(5).Info("GetWebMKSTicket") + + ticket, err := vm.AcquireTicket(vmCtx, string(types.VirtualMachineTicketTypeWebmks)) + if err != nil { + return "", err + } + + url := fmt.Sprintf("wss://%s:%d/ticket/%s", ticket.Host, ticket.Port, ticket.Ticket) + return EncryptWebMKS(pubKey, url) +} + +func EncryptWebMKS(pubKey string, plaintext string) (string, error) { + block, _ := pem.Decode([]byte(pubKey)) + if block == nil || block.Type != "PUBLIC KEY" { + return "", errors.New("failed to decode PEM block containing public key") + } + pub, err := x509.ParsePKCS1PublicKey(block.Bytes) + if err != nil { + return "", err + } + cipherbytes, err := rsa.EncryptOAEP(sha512.New(), rand.Reader, pub, []byte(plaintext), nil) + if err != nil { + return "", err + } + ciphertext := base64.StdEncoding.EncodeToString(cipherbytes) + return ciphertext, nil +} + +func DecryptWebMKS(privKey *rsa.PrivateKey, ciphertext string) (string, error) { + decoded, err := base64.StdEncoding.DecodeString(ciphertext) + if err != nil { + return "", err + } + decrypted, err := rsa.DecryptOAEP(sha512.New(), rand.Reader, privKey, decoded, nil) + if err != nil { + return "", err + } + return string(decrypted), nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go new file mode 100644 index 000000000..50a05cc72 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/webconsole_ticket_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "crypto/rsa" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var _ = Describe("Webconsole Ticket", func() { + + Context("EncryptWebMKS", func() { + var ( + privateKey *rsa.PrivateKey + publicKeyPem string + ) + + BeforeEach(func() { + privateKey, publicKeyPem = builder.WebConsoleRequestKeyPair() + }) + + It("Encrypts a string correctly", func() { + plaintext := "HelloWorld2" + ciphertext, err := virtualmachine.EncryptWebMKS(publicKeyPem, plaintext) + Expect(err).ShouldNot(HaveOccurred()) + decrypted, err := virtualmachine.DecryptWebMKS(privateKey, ciphertext) + Expect(err).ShouldNot(HaveOccurred()) + Expect(decrypted).To(Equal(plaintext)) + }) + + It("Error on invalid public key", func() { + plaintext := "HelloWorld3" + _, err := virtualmachine.EncryptWebMKS("invalid-pub-key", plaintext) + Expect(err).Should(HaveOccurred()) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go new file mode 100644 index 000000000..1546399e8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go @@ -0,0 +1,265 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "fmt" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/task" + vimTypes "github.com/vmware/govmomi/vim25/types" + apiEquality "k8s.io/apimachinery/pkg/api/equality" + ctrl "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/resources" +) + +const ( + // OvfEnvironmentTransportGuestInfo is the OVF transport type that uses + // GuestInfo. The other valid type is "iso". + OvfEnvironmentTransportGuestInfo = "com.vmware.guestInfo" +) + +type BootstrapData struct { + Data map[string]string + VAppData map[string]string + VAppExData map[string]map[string]string +} + +type TemplateRenderFunc func(string, string) string + +type BootstrapArgs struct { + BootstrapData + + TemplateRenderFn TemplateRenderFunc + NetworkResults network.NetworkInterfaceResults + Hostname string + DNSServers []string + SearchSuffixes []string +} + +func DoBootstrap( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo, + k8sClient ctrl.Client, + networkResults network.NetworkInterfaceResults, + bootstrapData BootstrapData) error { + + bootstrap := &vmCtx.VM.Spec.Bootstrap + cloudInit := bootstrap.CloudInit + linuxPrep := bootstrap.LinuxPrep + sysPrep := bootstrap.Sysprep + vAppConfig := bootstrap.VAppConfig + + bootstrapArgs, err := getBootstrapArgs(vmCtx, k8sClient, cloudInit != nil, networkResults, bootstrapData) + if err != nil { + return err + } + + if vAppConfig != nil { + // I think the intention was to only apply this to vAppData. Old code would apply it to entire + // Data map but for like SysPrep that data may be base64/gzip'd, and we'd do the template stuff + // prior to plain texting it. + bootstrapArgs.TemplateRenderFn = GetTemplateRenderFunc(vmCtx, bootstrapArgs) + } + + var configSpec *vimTypes.VirtualMachineConfigSpec + var customSpec *vimTypes.CustomizationSpec + + switch { + case cloudInit != nil: + configSpec, customSpec, err = BootStrapCloudInit(vmCtx, config, cloudInit, bootstrapArgs) + case linuxPrep != nil: + configSpec, customSpec, err = BootStrapLinuxPrep(vmCtx, config, linuxPrep, vAppConfig, bootstrapArgs) + case sysPrep != nil: + configSpec, customSpec, err = BootstrapSysPrep(vmCtx, config, sysPrep, vAppConfig, bootstrapArgs) + case vAppConfig != nil: + configSpec, customSpec, err = BootstrapVAppConfig(vmCtx, config, vAppConfig, bootstrapArgs) + default: + // Old code fell back to LinuxPrep. Is that really appropriate anymore? + linuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{HardwareClockIsUTC: true} + configSpec, customSpec, err = BootStrapLinuxPrep(vmCtx, config, linuxPrep, nil, bootstrapArgs) + } + + if err != nil { + return fmt.Errorf("failed to create bootstrap data: %w", err) + } + + if configSpec != nil { + err := doReconfigure(vmCtx, vcVM, configSpec) + if err != nil { + return fmt.Errorf("boostrap reconfigure failed: %w", err) + } + } + + if customSpec != nil { + err := doCustomize(vmCtx, vcVM, config, customSpec) + if err != nil { + return fmt.Errorf("boostrap customize failed: %w", err) + } + } + + return nil +} + +func getBootstrapArgs( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrl.Client, + isCloudInit bool, + networkResults network.NetworkInterfaceResults, + bootstrapData BootstrapData) (*BootstrapArgs, error) { + + bootstrapArgs := BootstrapArgs{ + BootstrapData: bootstrapData, + NetworkResults: networkResults, + Hostname: vmCtx.VM.Spec.Network.HostName, + } + + if bootstrapArgs.Hostname == "" { + bootstrapArgs.Hostname = vmCtx.VM.Name + } + + // If the VM is missing DNS info - that is, it did not specify DNS for the interfaces - populate that + // now from the SV global configuration. Note that the VM is probably OK as long as at least one + // interface has DNS info, but we would previously set it for every interface so keep doing that + // here. Similarly, we didn't populate SearchDomains for non-TKG VMs so we don't here either. This is + // all a little nuts & complicated and probably not correct for every situation. + isTKG := hasTKGLabels(vmCtx.VM.Labels) + missingDNSInfo := false + for _, r := range networkResults.Results { + if r.DHCP4 || r.DHCP6 { + continue + } + + if len(r.Nameservers) == 0 || (isTKG && len(r.SearchDomains) == 0) { + missingDNSInfo = true + break + } + } + + if missingDNSInfo { + nameservers, searchSuffixes, err := config.GetDNSInformationFromConfigMap(k8sClient) + if err != nil && ctrl.IgnoreNotFound(err) != nil { + // This ConfigMap doesn't exist in certain test envs. + return nil, err + } + + // GOSC will use these for its global config. + bootstrapArgs.DNSServers = nameservers + bootstrapArgs.SearchSuffixes = searchSuffixes + + if isCloudInit { + // Previously we would apply the global DNS config to every interface so do that here too. + for i := range networkResults.Results { + r := &networkResults.Results[i] + + if r.DHCP4 || r.DHCP6 { + continue + } + + if len(r.Nameservers) == 0 { + r.Nameservers = nameservers + } + if isTKG && len(r.SearchDomains) == 0 { + r.SearchDomains = searchSuffixes + } + } + } + } + + return &bootstrapArgs, nil +} + +func hasTKGLabels(vmLabels map[string]string) bool { + const ( + // CAPWClusterRoleLabelKey is the key for the label applied to a VM that was + // created by CAPW. + CAPWClusterRoleLabelKey = "capw.vmware.com/cluster.role" //nolint:gosec + + // CAPVClusterRoleLabelKey is the key for the label applied to a VM that was + // created by CAPV. + CAPVClusterRoleLabelKey = "capv.vmware.com/cluster.role" + ) + + _, ok := vmLabels[CAPWClusterRoleLabelKey] + if !ok { + _, ok = vmLabels[CAPVClusterRoleLabelKey] + } + return ok +} + +func doReconfigure( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + configSpec *vimTypes.VirtualMachineConfigSpec) error { + + defaultConfigSpec := &vimTypes.VirtualMachineConfigSpec{} + if !apiEquality.Semantic.DeepEqual(configSpec, defaultConfigSpec) { + vmCtx.Logger.Info("Customization Reconfigure", "configSpec", configSpec) + + if err := resources.NewVMFromObject(vcVM).Reconfigure(vmCtx, configSpec); err != nil { + vmCtx.Logger.Error(err, "customization reconfigure failed") + return err + } + } + + return nil +} + +func doCustomize( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + config *vimTypes.VirtualMachineConfigInfo, + customSpec *vimTypes.CustomizationSpec) error { + + if vmCtx.VM.Annotations[constants.VSphereCustomizationBypassKey] == constants.VSphereCustomizationBypassDisable { + vmCtx.Logger.Info("Skipping vsphere customization because of vsphere-customization bypass annotation") + return nil + } + + if IsCustomizationPendingExtraConfig(config.ExtraConfig) { + vmCtx.Logger.Info("Skipping customization because it is already pending") + // TODO: We should really determine if the pending customization is stale, clear it + // if so, and then re-customize. Otherwise, the Customize call could perpetually fail + // preventing power on. + return nil + } + + vmCtx.Logger.Info("Customizing VM", "customizationSpec", *customSpec) + if err := resources.NewVMFromObject(vcVM).Customize(vmCtx, *customSpec); err != nil { + // isCustomizationPendingExtraConfig() above is supposed to prevent this error, but + // handle it explicitly here just in case so VM reconciliation can proceed. + if !isCustomizationPendingError(err) { + return err + } + } + + return nil +} + +func IsCustomizationPendingExtraConfig(extraConfig []vimTypes.BaseOptionValue) bool { + for _, opt := range extraConfig { + if optValue := opt.GetOptionValue(); optValue != nil { + if optValue.Key == constants.GOSCPendingExtraConfigKey { + return optValue.Value.(string) != "" + } + } + } + return false +} + +func isCustomizationPendingError(err error) bool { + if te, ok := err.(task.Error); ok { + if _, ok := te.Fault().(*vimTypes.CustomizationPending); ok { + return true + } + } + return false +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go new file mode 100644 index 000000000..9e46536d4 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit.go @@ -0,0 +1,186 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "fmt" + "strings" + + "github.com/vmware/govmomi/vim25/types" + "gopkg.in/yaml.v2" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/internal" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +type CloudInitMetadata struct { + InstanceID string `yaml:"instance-id,omitempty"` + LocalHostname string `yaml:"local-hostname,omitempty"` + Hostname string `yaml:"hostname,omitempty"` + Network network.Netplan `yaml:"network,omitempty"` + PublicKeys string `yaml:"public-keys,omitempty"` +} + +func BootStrapCloudInit( + vmCtx context.VirtualMachineContextA2, + config *types.VirtualMachineConfigInfo, + cloudInitSpec *vmopv1.VirtualMachineBootstrapCloudInitSpec, + bsArgs *BootstrapArgs) (*types.VirtualMachineConfigSpec, *types.CustomizationSpec, error) { + + netPlan, err := network.NetPlanCustomization(bsArgs.NetworkResults) + if err != nil { + return nil, nil, fmt.Errorf("failed to create NetPlan customization: %w", err) + } + + sshPublicKeys := bsArgs.BootstrapData.Data["ssh-public-keys"] + if len(cloudInitSpec.SSHAuthorizedKeys) > 0 { + sshPublicKeys = strings.Join(cloudInitSpec.SSHAuthorizedKeys, "\n") + } + + metadata, err := GetCloudInitMetadata(string(vmCtx.VM.UID), bsArgs.Hostname, netPlan, sshPublicKeys) + if err != nil { + return nil, nil, err + } + + var userdata string + if cloudInitSpec.RawCloudConfig.Name != "" { + // Check for the 'user-data' key as per official contract and API documentation. + // Additionally, to support the cluster bootstrap data supplied by CAPBK's secret, + // we check for a 'value' key when 'user-data' is not supplied. + // The 'value' key lookup will eventually be deprecated. + for _, k := range []string{cloudInitSpec.RawCloudConfig.Key, "user-data", "value"} { + if k != "" { + userdata = bsArgs.BootstrapData.Data[k] + if userdata != "" { + break + } + } + } + + // NOTE: The old code didn't error out if userdata wasn't found, so keep going. + + } else { + return nil, nil, fmt.Errorf("TODO: inlined CloudConfig") + } + + var configSpec *types.VirtualMachineConfigSpec + var customSpec *types.CustomizationSpec + + switch vmCtx.VM.Annotations[constants.CloudInitTypeAnnotation] { + case constants.CloudInitTypeValueCloudInitPrep: + configSpec, customSpec, err = GetCloudInitPrepCustSpec(metadata, userdata) + case constants.CloudInitTypeValueGuestInfo, "": + fallthrough + default: + configSpec, err = GetCloudInitGuestInfoCustSpec(config, metadata, userdata) + } + + if err != nil { + return nil, nil, err + } + + return configSpec, customSpec, nil +} + +func GetCloudInitMetadata( + uid string, + hostname string, + netplan *network.Netplan, + sshPublicKeys string) (string, error) { + + metadata := &CloudInitMetadata{ + InstanceID: uid, + LocalHostname: hostname, + Hostname: hostname, + Network: *netplan, + PublicKeys: sshPublicKeys, + } + + metadataBytes, err := yaml.Marshal(metadata) + if err != nil { + return "", fmt.Errorf("yaml marshalling of cloud-init metadata failed: %w", err) + } + + return string(metadataBytes), nil +} + +func GetCloudInitPrepCustSpec( + metadata, userdata string) (*types.VirtualMachineConfigSpec, *types.CustomizationSpec, error) { + + if userdata != "" { + // Ensure the data is normalized first to plain-text. + plainText, err := util.TryToDecodeBase64Gzip([]byte(userdata)) + if err != nil { + return nil, nil, fmt.Errorf("decoding cloud-init prep userdata failed: %w", err) + } + + userdata = plainText + } + + // FIXME: This is the old behavior but isn't quite correct: we need current config, and only + // do this if the transport isn't already set. Otherwise, this always results in a Reconfigure. + configSpec := &types.VirtualMachineConfigSpec{ + VAppConfig: &types.VmConfigSpec{ + // Ensure the transport is guestInfo in case the VM does not have + // a CD-ROM device required to use the ISO transport. + OvfEnvironmentTransport: []string{OvfEnvironmentTransportGuestInfo}, + }, + } + + customSpec := &types.CustomizationSpec{ + Identity: &internal.CustomizationCloudinitPrep{ + Metadata: metadata, + Userdata: userdata, + }, + } + + return configSpec, customSpec, nil +} + +func GetCloudInitGuestInfoCustSpec( + config *types.VirtualMachineConfigInfo, + metadata, userdata string) (*types.VirtualMachineConfigSpec, error) { + + encodedMetadata, err := util.EncodeGzipBase64(metadata) + if err != nil { + return nil, fmt.Errorf("encoding cloud-init metadata failed: %w", err) + } + + extraConfig := map[string]string{ + constants.CloudInitGuestInfoMetadata: encodedMetadata, + constants.CloudInitGuestInfoMetadataEncoding: "gzip+base64", + } + + if userdata != "" { + // Ensure the data is normalized first to plain-text. + plainText, err := util.TryToDecodeBase64Gzip([]byte(userdata)) + if err != nil { + return nil, fmt.Errorf("decoding cloud-init userdata failed: %w", err) + } + + encodedUserdata, err := util.EncodeGzipBase64(plainText) + if err != nil { + return nil, fmt.Errorf("encoding cloud-init userdata failed: %w", err) + } + + extraConfig[constants.CloudInitGuestInfoUserdata] = encodedUserdata + extraConfig[constants.CloudInitGuestInfoUserdataEncoding] = "gzip+base64" + } + + // FIXME: This is the old behavior but isn't quite correct: we really need the current ExtraConfig, + // and then only add/update it if new or updated (so we don't customize with stale data). Also, + // always setting VAppConfigRemoved isn't correct: should only set it if the VM has vApp data. + // As-is we will always Reconfigure the VM. + configSpec := &types.VirtualMachineConfigSpec{} + configSpec.ExtraConfig = util.AppendNewExtraConfigValues(config.ExtraConfig, extraConfig) + // Remove the VAppConfig to ensure Cloud-Init inside the guest does not + // activate and prefer the OVF datasource over the VMware datasource. + configSpec.VAppConfigRemoved = types.NewBool(true) // FIXME + + return configSpec, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go new file mode 100644 index 000000000..7badf04c8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go @@ -0,0 +1,398 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + "encoding/base64" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + "gopkg.in/yaml.v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/internal" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("CloudInit Bootstrap", func() { + const ( + cloudInitMetadata = "cloud-init-metadata" + cloudInitUserdata = "cloud-init-userdata" + ) + + var ( + bsArgs vmlifecycle.BootstrapArgs + configInfo *types.VirtualMachineConfigInfo + + metaData string + userData string + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + bsArgs.Data = map[string]string{} + + // Set defaults. + metaData = cloudInitMetadata + userData = cloudInitUserdata + }) + + AfterEach(func() { + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + extraConfigToMap := func(input []types.BaseOptionValue) map[string]string { + output := make(map[string]string) + for _, opt := range input { + if optValue := opt.GetOptionValue(); optValue != nil { + // Only set string type values + if val, ok := optValue.Value.(string); ok { + output[optValue.Key] = val + } + } + } + return output + } + + // v1a1 tests really only tested the lower level functions individually. Those tests are ported after + // this Context, but we should focus more on testing via this just method. + Context("BootStrapCloudInit", func() { + var ( + configSpec *types.VirtualMachineConfigSpec + custSpec *types.CustomizationSpec + err error + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + cloudInitSpec *vmopv1.VirtualMachineBootstrapCloudInitSpec + ) + + BeforeEach(func() { + cloudInitSpec = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cloud-init-bootstrap-test", + Namespace: "test-ns", + UID: "my-vm-uuid", + Annotations: map[string]string{}, + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger(), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec, custSpec, err = vmlifecycle.BootStrapCloudInit( + vmCtx, + configInfo, + cloudInitSpec, + &bsArgs, + ) + }) + + Context("CloudInit Inlined Config", func() { + It("Returns TODO", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("TODO")) + }) + }) + + Context("CloudInit Data Config", func() { + BeforeEach(func() { + cloudInitSpec.RawCloudConfig.Name = "my-data" + cloudInitSpec.RawCloudConfig.Key = "my-key" + bsArgs.Data[cloudInitSpec.RawCloudConfig.Key] = cloudInitUserdata + }) + + Context("Via CloudInitPrep", func() { + BeforeEach(func() { + vmCtx.VM.Annotations[constants.CloudInitTypeAnnotation] = constants.CloudInitTypeValueCloudInitPrep + }) + + It("Returns success", func() { + Expect(err).ToNot(HaveOccurred()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec()).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport).To(HaveLen(1)) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport[0]).To(Equal(vmlifecycle.OvfEnvironmentTransportGuestInfo)) + + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).ToNot(BeEmpty()) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + + Context("Via GuestInfo", func() { + BeforeEach(func() { + vmCtx.VM.Annotations[constants.CloudInitTypeAnnotation] = constants.CloudInitTypeValueGuestInfo + }) + + It("Returns Success", func() { + Expect(err).ToNot(HaveOccurred()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfigRemoved).ToNot(BeNil()) + Expect(*configSpec.VAppConfigRemoved).To(BeTrue()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoMetadata)) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + + Expect(custSpec).To(BeNil()) + }) + + Context("Via CAPBK userdata in 'value' key", func() { + const otherUserData = cloudInitUserdata + "CAPBK" + + BeforeEach(func() { + bsArgs.Data[cloudInitSpec.RawCloudConfig.Key] = "" + cloudInitSpec.RawCloudConfig.Key = "" + bsArgs.Data["value"] = otherUserData + }) + + It("Returns success", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoMetadata)) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + + Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoUserdata)) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + + data, err := util.TryToDecodeBase64Gzip([]byte(extraConfig[constants.CloudInitGuestInfoUserdata])) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(Equal(otherUserData)) + }) + }) + }) + }) + }) + + Context("GetCloudInitMetadata", func() { + var ( + uid string + hostName string + netPlan *network.Netplan + sshPublicKeys string + + mdYaml string + err error + ) + + BeforeEach(func() { + uid = "my-uid" + hostName = "my-hostname" + netPlan = &network.Netplan{ + Version: 42, + Ethernets: map[string]network.NetplanEthernet{ + "eth0": { + SetName: "eth0", + }, + }, + } + sshPublicKeys = "my-ssh-key" + }) + + JustBeforeEach(func() { + mdYaml, err = vmlifecycle.GetCloudInitMetadata(uid, hostName, netPlan, sshPublicKeys) + }) + + It("DoIt", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(mdYaml).ToNot(BeEmpty()) + + ciMetadata := &vmlifecycle.CloudInitMetadata{} + Expect(yaml.Unmarshal([]byte(mdYaml), ciMetadata)).To(Succeed()) + + Expect(ciMetadata.InstanceID).To(Equal(uid)) + Expect(ciMetadata.Hostname).To(Equal(hostName)) + Expect(ciMetadata.PublicKeys).To(Equal(sshPublicKeys)) + Expect(ciMetadata.Network.Version).To(Equal(42)) + Expect(ciMetadata.Network.Ethernets).To(HaveKey("eth0")) + }) + }) + + Context("GetCloudInitGuestInfoCustSpec", func() { + var ( + configSpec *types.VirtualMachineConfigSpec + err error + ) + + JustBeforeEach(func() { + configSpec, err = vmlifecycle.GetCloudInitGuestInfoCustSpec(configInfo, metaData, userData) + }) + + Context("VAppConfig Disabled", func() { + It("Should disable the VAppConfig", func() { + Expect(err).ToNot(HaveOccurred()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfigRemoved).ToNot(BeNil()) + Expect(*configSpec.VAppConfigRemoved).To(BeTrue()) + }) + }) + + Context("No userdata", func() { + BeforeEach(func() { + userData = "" + }) + + It("ConfigSpec.ExtraConfig to only have metadata", func() { + Expect(configSpec).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(2)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + }) + }) + + Context("With userdata", func() { + It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + }) + }) + + Context("With base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + userData = base64.StdEncoding.EncodeToString([]byte(cloudInitUserdata)) + }) + + It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + }) + }) + + Context("With gzipped, base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + data, err := util.EncodeGzipBase64(cloudInitUserdata) + Expect(err).ToNot(HaveOccurred()) + userData = data + }) + + It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).ToNot(BeNil()) + + extraConfig := extraConfigToMap(configSpec.ExtraConfig) + Expect(extraConfig).To(HaveLen(4)) + Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRLS1OLUpJLEkEAAAA//8BAAD//weVSMoTAAAA")) + Expect(extraConfig[constants.CloudInitGuestInfoUserdataEncoding]).To(Equal("gzip+base64")) + }) + }) + + }) + + Context("GetCloudInitPrepCustSpec", func() { + var ( + custSpec *types.CustomizationSpec + ) + + JustBeforeEach(func() { + var configSpec *types.VirtualMachineConfigSpec + var err error + + configSpec, custSpec, err = vmlifecycle.GetCloudInitPrepCustSpec(metaData, userData) + Expect(err).ToNot(HaveOccurred()) + + // Validate that Cloud-Init Prep always uses the GuestInfo transport for the CI Prep's meta and user data. + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec()).ToNot(BeNil()) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport).To(HaveLen(1)) + Expect(configSpec.VAppConfig.GetVmConfigSpec().OvfEnvironmentTransport[0]).To(Equal(vmlifecycle.OvfEnvironmentTransportGuestInfo)) + }) + + Context("No userdata", func() { + BeforeEach(func() { + userData = "" + }) + + It("Cust spec to only have metadata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(BeEmpty()) + }) + }) + + Context("With userdata", func() { + It("Cust spec to have metadata and userdata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + + Context("With base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + userData = base64.StdEncoding.EncodeToString([]byte(cloudInitUserdata)) + }) + + It("Cust spec to have metadata and userdata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + + Context("With gzipped, base64-encoded userdata but no encoding specified", func() { + BeforeEach(func() { + data, err := util.EncodeGzipBase64(cloudInitUserdata) + Expect(err).ToNot(HaveOccurred()) + userData = data + }) + + It("Cust spec to have metadata and userdata", func() { + Expect(custSpec).ToNot(BeNil()) + cloudInitPrepSpec := custSpec.Identity.(*internal.CustomizationCloudinitPrep) + Expect(cloudInitPrepSpec.Metadata).To(Equal(cloudInitMetadata)) + Expect(cloudInitPrepSpec.Userdata).To(Equal(cloudInitUserdata)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go new file mode 100644 index 000000000..0f954fd0f --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep.go @@ -0,0 +1,55 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + "fmt" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +func BootStrapLinuxPrep( + ctx goctx.Context, + config *vimTypes.VirtualMachineConfigInfo, + linuxPrepSpec *vmopv1.VirtualMachineBootstrapLinuxPrepSpec, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { + + nicSettingMap, err := network.GuestOSCustomization(bsArgs.NetworkResults) + if err != nil { + return nil, nil, fmt.Errorf("failed to create GOSC NIC mappings: %w", err) + } + + customSpec := &vimTypes.CustomizationSpec{ + Identity: &vimTypes.CustomizationLinuxPrep{ + HostName: &vimTypes.CustomizationFixedName{ + Name: bsArgs.Hostname, + }, + TimeZone: linuxPrepSpec.TimeZone, + HwClockUTC: vimTypes.NewBool(linuxPrepSpec.HardwareClockIsUTC), + }, + GlobalIPSettings: vimTypes.CustomizationGlobalIPSettings{ + DnsSuffixList: bsArgs.SearchSuffixes, + DnsServerList: bsArgs.DNSServers, + }, + NicSettingMap: nicSettingMap, + } + + var configSpec *vimTypes.VirtualMachineConfigSpec + if vAppConfigSpec != nil { + configSpec = &vimTypes.VirtualMachineConfigSpec{} + configSpec.VAppConfig = GetOVFVAppConfigForConfigSpec( + config, + vAppConfigSpec, + bsArgs.BootstrapData.VAppData, + bsArgs.BootstrapData.VAppExData, + bsArgs.TemplateRenderFn) + } + + return configSpec, customSpec, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go new file mode 100644 index 000000000..06fcd4a12 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go @@ -0,0 +1,156 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("LinuxPrep Bootstrap", func() { + const ( + macAddr = "43-AB-B4-1B-7E-87" + ) + + var ( + bsArgs vmlifecycle.BootstrapArgs + configInfo *types.VirtualMachineConfigInfo + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + bsArgs.Data = map[string]string{} + }) + + AfterEach(func() { + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + Context("BootStrapLinuxPrep", func() { + + var ( + configSpec *types.VirtualMachineConfigSpec + custSpec *types.CustomizationSpec + err error + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + linuxPrepSpec *vmopv1.VirtualMachineBootstrapLinuxPrepSpec + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec + ) + + BeforeEach(func() { + linuxPrepSpec = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} + vAppConfigSpec = nil + + bsArgs.Hostname = "my-hostname" + bsArgs.SearchSuffixes = []string{"suffix1", "suffix2"} + bsArgs.NetworkResults.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: "192.168.1.1", + IPCIDR: "192.168.1.10/24", + IsIPv4: true, + }, + }, + }, + } + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "linux-prep-bootstrap-test", + Namespace: "test-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger(), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec, custSpec, err = vmlifecycle.BootStrapLinuxPrep( + vmCtx, + configInfo, + linuxPrepSpec, + vAppConfigSpec, + &bsArgs, + ) + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).To(BeNil()) + + Expect(custSpec).ToNot(BeNil()) + Expect(custSpec.GlobalIPSettings.DnsServerList).To(Equal(bsArgs.DNSServers)) + Expect(custSpec.GlobalIPSettings.DnsSuffixList).To(Equal(bsArgs.SearchSuffixes)) + + linuxSpec := custSpec.Identity.(*types.CustomizationLinuxPrep) + hostName := linuxSpec.HostName.(*types.CustomizationFixedName).Name + Expect(hostName).To(Equal(bsArgs.Hostname)) + Expect(linuxSpec.TimeZone).To(Equal(linuxPrepSpec.TimeZone)) + Expect(linuxSpec.HwClockUTC).ToNot(BeNil()) + Expect(*linuxSpec.HwClockUTC).To(Equal(linuxPrepSpec.HardwareClockIsUTC)) + + Expect(custSpec.NicSettingMap).To(HaveLen(len(bsArgs.NetworkResults.Results))) + Expect(custSpec.NicSettingMap[0].MacAddress).To(Equal(macAddr)) + }) + + Context("when has vAppConfig", func() { + const key, value = "fooKey", "fooValue" + + BeforeEach(func() { + configInfo.VAppConfig = &types.VmConfigInfo{ + Property: []types.VAppPropertyInfo{ + { + Id: key, + Value: "should-change", + UserConfigurable: pointer.Bool(true), + }, + }, + } + + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{Value: pointer.String(value)}, + }, + }, + } + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(custSpec).ToNot(BeNil()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + vmCs := configSpec.VAppConfig.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go new file mode 100644 index 000000000..a8fe2baea --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go @@ -0,0 +1,79 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + "fmt" + + vimTypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/apimachinery/pkg/api/equality" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/sysprep" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" +) + +func BootstrapSysPrep( + ctx goctx.Context, + config *vimTypes.VirtualMachineConfigInfo, + sysPrepSpec *vmopv1.VirtualMachineBootstrapSysprepSpec, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { + + var data string + + if equality.Semantic.DeepEqual(sysPrepSpec.Sysprep, sysprep.Sysprep{}) { + var err error + + key := "unattend" + if sysPrepSpec.RawSysprep.Key != "" { + key = sysPrepSpec.RawSysprep.Key + } + + data = bsArgs.BootstrapData.Data[key] + if data == "" { + return nil, nil, fmt.Errorf("no Sysprep XML data with key %q", key) + } + + // Ensure the data is normalized first to plain-text. + data, err = util.TryToDecodeBase64Gzip([]byte(data)) + if err != nil { + return nil, nil, fmt.Errorf("decoding Sysprep unattend XML failed: %w", err) + } + + } else { + return nil, nil, fmt.Errorf("TODO: inlined Sysprep") + } + + nicSettingMap, err := network.GuestOSCustomization(bsArgs.NetworkResults) + if err != nil { + return nil, nil, fmt.Errorf("failed to create GSOC adapter mappings: %w", err) + } + + customSpec := &vimTypes.CustomizationSpec{ + Identity: &vimTypes.CustomizationSysprepText{ + Value: data, + }, + GlobalIPSettings: vimTypes.CustomizationGlobalIPSettings{ + DnsSuffixList: bsArgs.SearchSuffixes, + DnsServerList: bsArgs.DNSServers, + }, + NicSettingMap: nicSettingMap, + } + + var configSpec *vimTypes.VirtualMachineConfigSpec + if vAppConfigSpec != nil { + configSpec = &vimTypes.VirtualMachineConfigSpec{} + configSpec.VAppConfig = GetOVFVAppConfigForConfigSpec( + config, + vAppConfigSpec, + bsArgs.BootstrapData.VAppData, + bsArgs.BootstrapData.VAppExData, + bsArgs.TemplateRenderFn) + } + + return configSpec, customSpec, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go new file mode 100644 index 000000000..2a61f0ac7 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("SysPrep Bootstrap", func() { + const ( + macAddr = "43-AB-B4-1B-7E-87" + ) + + var ( + bsArgs vmlifecycle.BootstrapArgs + configInfo *types.VirtualMachineConfigInfo + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + bsArgs.Data = map[string]string{} + }) + + AfterEach(func() { + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + Context("BootStrapSysPrep", func() { + const unattendXML = "dummy-unattend-xml" + + var ( + configSpec *types.VirtualMachineConfigSpec + custSpec *types.CustomizationSpec + err error + + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + sysPrepSpec *vmopv1.VirtualMachineBootstrapSysprepSpec + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec + ) + + BeforeEach(func() { + sysPrepSpec = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + vAppConfigSpec = nil + + bsArgs.Data["unattend"] = unattendXML + bsArgs.SearchSuffixes = []string{"suffix1", "suffix2"} + bsArgs.NetworkResults.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: "192.168.1.1", + IPCIDR: "192.168.1.10/24", + IsIPv4: true, + }, + }, + }, + } + + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sys-prep-bootstrap-test", + Namespace: "test-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger(), + VM: vm, + } + }) + + JustBeforeEach(func() { + configSpec, custSpec, err = vmlifecycle.BootstrapSysPrep( + vmCtx, + configInfo, + sysPrepSpec, + vAppConfigSpec, + &bsArgs, + ) + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(configSpec).To(BeNil()) + + Expect(custSpec).ToNot(BeNil()) + Expect(custSpec.GlobalIPSettings.DnsServerList).To(Equal(bsArgs.DNSServers)) + Expect(custSpec.GlobalIPSettings.DnsSuffixList).To(Equal(bsArgs.SearchSuffixes)) + + sysPrepText := custSpec.Identity.(*types.CustomizationSysprepText) + Expect(sysPrepText.Value).To(Equal(unattendXML)) + + Expect(custSpec.NicSettingMap).To(HaveLen(len(bsArgs.NetworkResults.Results))) + Expect(custSpec.NicSettingMap[0].MacAddress).To(Equal(macAddr)) + }) + + Context("when has vAppConfig", func() { + const key, value = "fooKey", "fooValue" + + BeforeEach(func() { + configInfo.VAppConfig = &types.VmConfigInfo{ + Property: []types.VAppPropertyInfo{ + { + Id: key, + Value: "should-change", + UserConfigurable: pointer.Bool(true), + }, + }, + } + + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{Value: pointer.String(value)}, + }, + }, + } + }) + + It("should return expected customization spec", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(custSpec).ToNot(BeNil()) + + Expect(configSpec).ToNot(BeNil()) + Expect(configSpec.VAppConfig).ToNot(BeNil()) + vmCs := configSpec.VAppConfig.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go new file mode 100644 index 000000000..e80ee1042 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata.go @@ -0,0 +1,453 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "bytes" + "errors" + "fmt" + "net" + "strings" + "text/template" + + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +func GetTemplateRenderFunc( + vmCtx context.VirtualMachineContextA2, + bsArgs *BootstrapArgs, +) TemplateRenderFunc { + + // There is a lot of duplication here, especially since the "template" types are the same in v1a1 + // and v1a2. We've conflated a lot of things here making this all a little nuts. + + networkDevicesStatusV1A1 := toTemplateNetworkStatusV1A1(bsArgs) + networkStatusV1A1 := v1alpha1.NetworkStatus{ + Devices: networkDevicesStatusV1A1, + Nameservers: bsArgs.DNSServers, + } + + networkDevicesStatusV1A2 := toTemplateNetworkStatus(bsArgs) + networkStatusV1A2 := v1alpha2.NetworkStatus{ + Devices: networkDevicesStatusV1A2, + Nameservers: bsArgs.DNSServers, + } + + // Oh dear. The VM itself really should not have been included here. + v1a1VM := &v1alpha1.VirtualMachine{} + _ = v1a1VM.ConvertFrom(vmCtx.VM) + + templateData := struct { + V1alpha1 v1alpha1.VirtualMachineTemplate + V1alpha2 v1alpha2.VirtualMachineTemplate + }{ + V1alpha1: v1alpha1.VirtualMachineTemplate{ + Net: networkStatusV1A1, + VM: v1a1VM, + }, + V1alpha2: v1alpha2.VirtualMachineTemplate{ + Net: networkStatusV1A2, + VM: vmCtx.VM, + }, + } + + v1a1FuncMap := v1a1TemplateFunctions(networkStatusV1A1, networkDevicesStatusV1A1) + v1a2FuncMap := v1a2TemplateFunctions(networkStatusV1A2, networkDevicesStatusV1A2) + + // Include both but should probably leave out v1a2 if we can identify this was originally a v1a1 VM. + funcMap := template.FuncMap{} + for k, v := range v1a1FuncMap { + funcMap[k] = v + } + for k, v := range v1a2FuncMap { + funcMap[k] = v + } + + // Skip parsing when encountering escape character('\{',"\}") + normalizeStr := func(str string) string { + if strings.Contains(str, "\\{") || strings.Contains(str, "\\}") { + str = strings.ReplaceAll(str, "\\{", "{") + str = strings.ReplaceAll(str, "\\}", "}") + } + return str + } + + // TODO: Don't log, return errors instead. + renderTemplate := func(name, templateStr string) string { + templ, err := template.New(name).Funcs(funcMap).Parse(templateStr) + if err != nil { + vmCtx.Logger.Error(err, "failed to parse template", "templateStr", templateStr) + return normalizeStr(templateStr) + } + var doc bytes.Buffer + err = templ.Execute(&doc, &templateData) + if err != nil { + vmCtx.Logger.Error(err, "failed to execute template", "templateStr", templateStr) + return normalizeStr(templateStr) + } + return normalizeStr(doc.String()) + } + + return renderTemplate +} + +func v1a1TemplateFunctions( + networkStatusV1A1 v1alpha1.NetworkStatus, + networkDevicesStatusV1A1 []v1alpha1.NetworkDeviceStatus) map[string]any { + + // Get the first IP address from the first NIC. + v1alpha1FirstIP := func() (string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A1[0].IPAddresses[0], nil + } + + // Get the first NIC's MAC address. + v1alpha1FirstNicMacAddr := func() (string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A1[0].MacAddress, nil + } + + // Get the first IP address from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha1FirstIPFromNIC := func(index int) (string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A1) { + return "", errors.New("index out of bound") + } + return networkDevicesStatusV1A1[index].IPAddresses[0], nil + } + + // Get all IP addresses from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha1IPsFromNIC := func(index int) ([]string, error) { + if len(networkDevicesStatusV1A1) == 0 { + return []string{""}, errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A1) { + return []string{""}, errors.New("index out of bound") + } + return networkDevicesStatusV1A1[index].IPAddresses, nil + } + + // Format the first occurred count of nameservers with specific delimiter + // A negative count number would mean format all nameservers + v1alpha1FormatNameservers := func(count int, delimiter string) (string, error) { + var nameservers []string + if len(networkStatusV1A1.Nameservers) == 0 { + return "", errors.New("no available nameservers, check with VI admin") + } + if count < 0 || count >= len(networkStatusV1A1.Nameservers) { + nameservers = networkStatusV1A1.Nameservers + return strings.Join(nameservers, delimiter), nil + } + nameservers = networkStatusV1A1.Nameservers[:count] + return strings.Join(nameservers, delimiter), nil + } + + // Get subnet mask from a CIDR notation IP address and prefix length + // if IP address and prefix length not valid, throw an error and template string won't be parsed + v1alpha1SubnetMask := func(cidr string) (string, error) { + _, ipv4Net, err := net.ParseCIDR(cidr) + if err != nil { + return "", err + } + netmask := fmt.Sprintf("%d.%d.%d.%d", ipv4Net.Mask[0], ipv4Net.Mask[1], ipv4Net.Mask[2], ipv4Net.Mask[3]) + return netmask, nil + } + + // Format an IP address with default netmask CIDR + // if IP not valid, throw an error and template string won't be parsed + v1alpha1IP := func(IP string) (string, error) { + if net.ParseIP(IP) == nil { + return "", errors.New("input IP address not valid") + } + defaultMask := net.ParseIP(IP).DefaultMask() + ones, _ := defaultMask.Size() + expectedCidrNotation := IP + "/" + fmt.Sprintf("%d", int32(ones)) + return expectedCidrNotation, nil + } + + // Format an IP address with network length(eg. /24) or decimal + // notation (eg. 255.255.255.0). Format an IP/CIDR with updated mask. + // An empty mask causes just the IP to be returned. + v1alpha1FormatIP := func(s string, mask string) (string, error) { + // Get the IP address for the input string. + ip, _, err := net.ParseCIDR(s) + if err != nil { + ip = net.ParseIP(s) + if ip == nil { + return "", fmt.Errorf("input IP address not valid") + } + } + // Store the IP as a string back into s. + s = ip.String() + + // If no mask was provided then return just the IP. + if mask == "" { + return s, nil + } + + // The provided mask is a network length. + if strings.HasPrefix(mask, "/") { + s += mask + if _, _, err := net.ParseCIDR(s); err != nil { + return "", err + } + return s, nil + } + + // The provided mask is subnet mask. + maskIP := net.ParseIP(mask) + if maskIP == nil { + return "", fmt.Errorf("mask is an invalid IP") + } + + maskIPBytes := maskIP.To4() + if len(maskIPBytes) == 0 { + maskIPBytes = maskIP.To16() + } + + ipNet := net.IPNet{ + IP: ip, + Mask: net.IPMask(maskIPBytes), + } + s = ipNet.String() + + // Validate the ipNet is an IP/CIDR + if _, _, err := net.ParseCIDR(s); err != nil { + return "", fmt.Errorf("invalid ip net: %s", s) + } + + return s, nil + } + + return template.FuncMap{ + constants.V1alpha1FirstIP: v1alpha1FirstIP, + constants.V1alpha1FirstNicMacAddr: v1alpha1FirstNicMacAddr, + constants.V1alpha1FirstIPFromNIC: v1alpha1FirstIPFromNIC, + constants.V1alpha1IPsFromNIC: v1alpha1IPsFromNIC, + constants.V1alpha1FormatNameservers: v1alpha1FormatNameservers, + // These are more util function that we've conflated version namespaces. + constants.V1alpha1SubnetMask: v1alpha1SubnetMask, + constants.V1alpha1IP: v1alpha1IP, + constants.V1alpha1FormatIP: v1alpha1FormatIP, + } +} + +func toTemplateNetworkStatus(bsArgs *BootstrapArgs) []v1alpha2.NetworkDeviceStatus { + networkDevicesStatus := make([]v1alpha2.NetworkDeviceStatus, 0, len(bsArgs.NetworkResults.Results)) + + for _, result := range bsArgs.NetworkResults.Results { + // When using Sysprep, the MAC address must be in the format of "-". + // CloudInit normalizes it again to ":" when adding it to the netplan. + macAddr := strings.ReplaceAll(result.MacAddress, ":", "-") + + status := v1alpha2.NetworkDeviceStatus{ + MacAddress: macAddr, + } + + for _, ipConfig := range result.IPConfigs { + // We mostly only did IPv4 before so keep that going. + if ipConfig.IsIPv4 { + if status.Gateway4 == "" { + status.Gateway4 = ipConfig.Gateway + } + + status.IPAddresses = append(status.IPAddresses, ipConfig.IPCIDR) + } + } + + networkDevicesStatus = append(networkDevicesStatus, status) + } + + return networkDevicesStatus +} + +// This is basically identical to v1a1TemplateFunctions. +func v1a2TemplateFunctions( + networkStatusV1A2 v1alpha2.NetworkStatus, + networkDevicesStatusV1A2 []v1alpha2.NetworkDeviceStatus) map[string]any { + + // Get the first IP address from the first NIC. + v1alpha2FirstIP := func() (string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A2[0].IPAddresses[0], nil + } + + // Get the first NIC's MAC address. + v1alpha2FirstNicMacAddr := func() (string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + return networkDevicesStatusV1A2[0].MacAddress, nil + } + + // Get the first IP address from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha2FirstIPFromNIC := func(index int) (string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return "", errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A2) { + return "", errors.New("index out of bound") + } + return networkDevicesStatusV1A2[index].IPAddresses[0], nil + } + + // Get all IP addresses from the ith NIC. + // if index out of bound, throw an error and template string won't be parsed + v1alpha2IPsFromNIC := func(index int) ([]string, error) { + if len(networkDevicesStatusV1A2) == 0 { + return []string{""}, errors.New("no available network device, check with VI admin") + } + if index >= len(networkDevicesStatusV1A2) { + return []string{""}, errors.New("index out of bound") + } + return networkDevicesStatusV1A2[index].IPAddresses, nil + } + + // Format the first occurred count of nameservers with specific delimiter + // A negative count number would mean format all nameservers + v1alpha2FormatNameservers := func(count int, delimiter string) (string, error) { + var nameservers []string + if len(networkStatusV1A2.Nameservers) == 0 { + return "", errors.New("no available nameservers, check with VI admin") + } + if count < 0 || count >= len(networkStatusV1A2.Nameservers) { + nameservers = networkStatusV1A2.Nameservers + return strings.Join(nameservers, delimiter), nil + } + nameservers = networkStatusV1A2.Nameservers[:count] + return strings.Join(nameservers, delimiter), nil + } + + // Get subnet mask from a CIDR notation IP address and prefix length + // if IP address and prefix length not valid, throw an error and template string won't be parsed + v1alpha2SubnetMask := func(cidr string) (string, error) { + _, ipv4Net, err := net.ParseCIDR(cidr) + if err != nil { + return "", err + } + netmask := fmt.Sprintf("%d.%d.%d.%d", ipv4Net.Mask[0], ipv4Net.Mask[1], ipv4Net.Mask[2], ipv4Net.Mask[3]) + return netmask, nil + } + + // Format an IP address with default netmask CIDR + // if IP not valid, throw an error and template string won't be parsed + v1alpha2IP := func(IP string) (string, error) { + if net.ParseIP(IP) == nil { + return "", errors.New("input IP address not valid") + } + defaultMask := net.ParseIP(IP).DefaultMask() + ones, _ := defaultMask.Size() + expectedCidrNotation := IP + "/" + fmt.Sprintf("%d", int32(ones)) + return expectedCidrNotation, nil + } + + // Format an IP address with network length(eg. /24) or decimal + // notation (eg. 255.255.255.0). Format an IP/CIDR with updated mask. + // An empty mask causes just the IP to be returned. + v1alpha2FormatIP := func(s string, mask string) (string, error) { + // Get the IP address for the input string. + ip, _, err := net.ParseCIDR(s) + if err != nil { + ip = net.ParseIP(s) + if ip == nil { + return "", fmt.Errorf("input IP address not valid") + } + } + // Store the IP as a string back into s. + s = ip.String() + + // If no mask was provided then return just the IP. + if mask == "" { + return s, nil + } + + // The provided mask is a network length. + if strings.HasPrefix(mask, "/") { + s += mask + if _, _, err := net.ParseCIDR(s); err != nil { + return "", err + } + return s, nil + } + + // The provided mask is subnet mask. + maskIP := net.ParseIP(mask) + if maskIP == nil { + return "", fmt.Errorf("mask is an invalid IP") + } + + maskIPBytes := maskIP.To4() + if len(maskIPBytes) == 0 { + maskIPBytes = maskIP.To16() + } + + ipNet := net.IPNet{ + IP: ip, + Mask: net.IPMask(maskIPBytes), + } + s = ipNet.String() + + // Validate the ipNet is an IP/CIDR + if _, _, err := net.ParseCIDR(s); err != nil { + return "", fmt.Errorf("invalid ip net: %s", s) + } + + return s, nil + } + + return template.FuncMap{ + constants.V1alpha2FirstIP: v1alpha2FirstIP, + constants.V1alpha2FirstNicMacAddr: v1alpha2FirstNicMacAddr, + constants.V1alpha2FirstIPFromNIC: v1alpha2FirstIPFromNIC, + constants.V1alpha2IPsFromNIC: v1alpha2IPsFromNIC, + constants.V1alpha2FormatNameservers: v1alpha2FormatNameservers, + // These are more util function that we've conflated version namespaces. + constants.V1alpha2SubnetMask: v1alpha2SubnetMask, + constants.V1alpha2IP: v1alpha2IP, + constants.V1alpha2FormatIP: v1alpha2FormatIP, + } +} + +func toTemplateNetworkStatusV1A1(bsArgs *BootstrapArgs) []v1alpha1.NetworkDeviceStatus { + networkDevicesStatus := make([]v1alpha1.NetworkDeviceStatus, 0, len(bsArgs.NetworkResults.Results)) + + for _, result := range bsArgs.NetworkResults.Results { + // When using Sysprep, the MAC address must be in the format of "-". + // CloudInit normalizes it again to ":" when adding it to the netplan. + macAddr := strings.ReplaceAll(result.MacAddress, ":", "-") + + status := v1alpha1.NetworkDeviceStatus{ + MacAddress: macAddr, + } + + for _, ipConfig := range result.IPConfigs { + // We mostly only did IPv4 before so keep that going. + if ipConfig.IsIPv4 { + if status.Gateway4 == "" { + status.Gateway4 = ipConfig.Gateway + } + + status.IPAddresses = append(status.IPAddresses, ipConfig.IPCIDR) + } + } + + networkDevicesStatus = append(networkDevicesStatus, status) + } + + return networkDevicesStatus +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go new file mode 100644 index 000000000..18c6a6b22 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_templatedata_test.go @@ -0,0 +1,221 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + goctx "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("TemplateVMMetadata", func() { + + const ( + ip1 = "192.168.1.37" + ip1Cidr = ip1 + "/24" + ip2 = "192.168.10.48" + ip2Cidr = ip2 + "/24" + gateway1 = "192.168.1.1" + gateway2 = "192.168.10.1" + nameserver1 = "8.8.8.8" + nameserver2 = "1.1.1.1" + macAddr1 = "8a-cb-a0-1d-8d-c4" + macAddr2 = "00-cb-30-42-05-89" + ) + + var ( + vmCtx context.VirtualMachineContextA2 + vm *vmopv1.VirtualMachine + bsArgs *vmlifecycle.BootstrapArgs + ) + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-vm", + Namespace: "dummy-ns", + }, + } + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.Background(), + Logger: suite.GetLogger().WithName("bootstrap-template-tests"), + VM: vm, + } + + bsArgs = &vmlifecycle.BootstrapArgs{} + bsArgs.Data = make(map[string]string) + bsArgs.DNSServers = []string{nameserver1, nameserver2} + bsArgs.NetworkResults.Results = []network.NetworkInterfaceResult{ + { + MacAddress: macAddr1, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: gateway1, + IPCIDR: ip1Cidr, + IsIPv4: true, + }, + }, + }, + { + MacAddress: macAddr2, + IPConfigs: []network.NetworkInterfaceIPConfig{ + { + Gateway: gateway2, + IPCIDR: ip2Cidr, + IsIPv4: true, + }, + }, + }, + } + }) + + Context("Template Functions", func() { + DescribeTable("v1alpha1 template functions", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("first_cidrIp", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}", ip1Cidr), + Entry("second_cidrIp", "{{ (index (index .V1alpha1.Net.Devices 1).IPAddresses 0) }}", ip2Cidr), + Entry("first_gateway", "{{ (index .V1alpha1.Net.Devices 0).Gateway4 }}", gateway1), + Entry("second_gateway", "{{ (index .V1alpha1.Net.Devices 1).Gateway4 }}", gateway2), + Entry("nameserver", "{{ (index .V1alpha1.Net.Nameservers 0) }}", nameserver1), + Entry("first_macAddr", "{{ (index .V1alpha1.Net.Devices 0).MacAddress }}", macAddr1), + Entry("second_macAddr", "{{ (index .V1alpha1.Net.Devices 1).MacAddress }}", macAddr2), + Entry("name", "{{ .V1alpha1.VM.Name }}", "dummy-vm"), + ) + + DescribeTable("v1alpha2 template functions", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("first_cidrIp", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}", ip1Cidr), + Entry("second_cidrIp", "{{ (index (index .V1alpha2.Net.Devices 1).IPAddresses 0) }}", ip2Cidr), + Entry("first_gateway", "{{ (index .V1alpha2.Net.Devices 0).Gateway4 }}", gateway1), + Entry("second_gateway", "{{ (index .V1alpha2.Net.Devices 1).Gateway4 }}", gateway2), + Entry("nameserver", "{{ (index .V1alpha2.Net.Nameservers 0) }}", nameserver1), + Entry("first_macAddr", "{{ (index .V1alpha2.Net.Devices 0).MacAddress }}", macAddr1), + Entry("second_macAddr", "{{ (index .V1alpha2.Net.Devices 1).MacAddress }}", macAddr2), + Entry("name", "{{ .V1alpha2.VM.Name }}", "dummy-vm"), + ) + }) + + Context("Function names", func() { + DescribeTable("v1alpha1 constant names", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("cidr_ip1", "{{ "+constants.V1alpha1FirstIP+" }}", ip1Cidr), + Entry("cidr_ip2", "{{ "+constants.V1alpha1FirstIPFromNIC+" 1 }}", ip2Cidr), + Entry("cidr_ip3", "{{ ("+constants.V1alpha1IP+" \"192.168.1.37\") }}", ip1Cidr), + Entry("cidr_ip4", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37\" \"/24\") }}", ip1Cidr), + Entry("cidr_ip5", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip6", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37/28\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip7", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37/28\" \"/24\") }}", ip1Cidr), + Entry("ip1", "{{ "+constants.V1alpha1FormatIP+" "+constants.V1alpha1FirstIP+" \"\" }}", ip1), + Entry("ip2", "{{ "+constants.V1alpha1FormatIP+" \"192.168.1.37/28\" \"\" }}", ip1), + Entry("ips_1", "{{ "+constants.V1alpha1IPsFromNIC+" 0 }}", fmt.Sprint([]string{ip1Cidr})), + Entry("subnetmask", "{{ "+constants.V1alpha1SubnetMask+" \"192.168.1.37/26\" }}", "255.255.255.192"), + Entry("firstNicMacAddr", "{{ "+constants.V1alpha1FirstNicMacAddr+" }}", macAddr1), + Entry("formatted_nameserver1", "{{ "+constants.V1alpha1FormatNameservers+" 1 \"-\"}}", nameserver1), + Entry("formatted_nameserver2", "{{ "+constants.V1alpha1FormatNameservers+" -1 \"-\"}}", nameserver1+"-"+nameserver2), + ) + + DescribeTable("v1alpha2 constant names", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("cidr_ip1", "{{ "+constants.V1alpha2FirstIP+" }}", ip1Cidr), + Entry("cidr_ip2", "{{ "+constants.V1alpha2FirstIPFromNIC+" 1 }}", ip2Cidr), + Entry("cidr_ip3", "{{ ("+constants.V1alpha2IP+" \"192.168.1.37\") }}", ip1Cidr), + Entry("cidr_ip4", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37\" \"/24\") }}", ip1Cidr), + Entry("cidr_ip5", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip6", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37/28\" \"255.255.255.0\") }}", ip1Cidr), + Entry("cidr_ip7", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37/28\" \"/24\") }}", ip1Cidr), + Entry("ip1", "{{ "+constants.V1alpha2FormatIP+" "+constants.V1alpha1FirstIP+" \"\" }}", ip1), + Entry("ip2", "{{ "+constants.V1alpha2FormatIP+" \"192.168.1.37/28\" \"\" }}", ip1), + Entry("ips_1", "{{ "+constants.V1alpha2IPsFromNIC+" 0 }}", fmt.Sprint([]string{ip1Cidr})), + Entry("subnetmask", "{{ "+constants.V1alpha2SubnetMask+" \"192.168.1.37/26\" }}", "255.255.255.192"), + Entry("firstNicMacAddr", "{{ "+constants.V1alpha2FirstNicMacAddr+" }}", macAddr1), + Entry("formatted_nameserver1", "{{ "+constants.V1alpha2FormatNameservers+" 1 \"-\"}}", nameserver1), + Entry("formatted_nameserver2", "{{ "+constants.V1alpha2FormatNameservers+" -1 \"-\"}}", nameserver1+"-"+nameserver2), + ) + }) + + Context("Invalid template names", func() { + DescribeTable("returns the original text", + func(str string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(str)) + }, + Entry("ip1", "{{ "+constants.V1alpha1IP+" \"192.1.0\" }}"), + Entry("ip2", "{{ "+constants.V1alpha1FirstIPFromNIC+" 5 }}"), + Entry("ips_1", "{{ "+constants.V1alpha1IPsFromNIC+" 5 }}"), + Entry("cidr_ip1", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1.37\" \"127.255.255.255\") }}"), + Entry("cidr_ip2", "{{ ("+constants.V1alpha1FormatIP+" \"192.168.1\" \"255.0.0.0\") }}"), + Entry("gateway", "{{ (index .V1alpha1.Net.NetworkInterfaces ).Gateway }}"), + Entry("nameserver", "{{ (index .V1alpha1.Net.NameServers 0) }}"), + ) + + DescribeTable("returns the original text, v1a2 style", + func(str string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(str)) + }, + Entry("ip1", "{{ "+constants.V1alpha2IP+" \"192.1.0\" }}"), + Entry("ip2", "{{ "+constants.V1alpha2FirstIPFromNIC+" 5 }}"), + Entry("ips_1", "{{ "+constants.V1alpha2IPsFromNIC+" 5 }}"), + Entry("cidr_ip1", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1.37\" \"127.255.255.255\") }}"), + Entry("cidr_ip2", "{{ ("+constants.V1alpha2FormatIP+" \"192.168.1\" \"255.0.0.0\") }}"), + Entry("gateway", "{{ (index .V1alpha2.Net.NetworkInterfaces ).Gateway }}"), + Entry("nameserver", "{{ (index .V1alpha2.Net.NameServers 0) }}"), + ) + }) + + Context("String has escape characters", func() { + DescribeTable("return one level of escaped removed", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("skip_data1", "\\{\\{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data2", "\\{\\{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data3", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data4", "skip \\{\\{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) \\}\\}", "skip {{ (index (index .V1alpha1.Net.Devices 0).IPAddresses 0) }}"), + ) + + DescribeTable("return one level of escaped removed, v1a2 style", + func(str, expected string) { + fn := vmlifecycle.GetTemplateRenderFunc(vmCtx, bsArgs) + out := fn("", str) + Expect(out).To(Equal(expected)) + }, + Entry("skip_data1", "\\{\\{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data2", "\\{\\{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data3", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) \\}\\}", "{{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + Entry("skip_data4", "skip \\{\\{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) \\}\\}", "skip {{ (index (index .V1alpha2.Net.Devices 0).IPAddresses 0) }}"), + ) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go new file mode 100644 index 000000000..11d3762cc --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_test.go @@ -0,0 +1,51 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("Customization utils", func() { + Context("IsPending", func() { + var extraConfig []vimTypes.BaseOptionValue + var pending bool + + BeforeEach(func() { + extraConfig = nil + }) + + JustBeforeEach(func() { + pending = vmlifecycle.IsCustomizationPendingExtraConfig(extraConfig) + }) + + Context("Empty ExtraConfig", func() { + It("not pending", func() { + Expect(pending).To(BeFalse()) + }) + }) + + Context("ExtraConfig with pending key", func() { + BeforeEach(func() { + extraConfig = append(extraConfig, &vimTypes.OptionValue{ + Key: constants.GOSCPendingExtraConfigKey, + Value: "/foo/bar", + }) + }) + + It("is pending", func() { + Expect(pending).To(BeTrue()) + }) + }) + }) +}) + +// TODO: We should at least a few basic DoBootstrap() tests so we test the overall +// Reconfigure/Customize flow but the old code didn't. diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go new file mode 100644 index 000000000..241f28928 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig.go @@ -0,0 +1,108 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + + vimTypes "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" +) + +func BootstrapVAppConfig( + ctx goctx.Context, + config *vimTypes.VirtualMachineConfigInfo, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { + + configSpec := &vimTypes.VirtualMachineConfigSpec{} + configSpec.VAppConfig = GetOVFVAppConfigForConfigSpec( + config, + vAppConfigSpec, + bsArgs.BootstrapData.VAppData, + bsArgs.BootstrapData.VAppExData, + bsArgs.TemplateRenderFn) + + return configSpec, nil, nil +} + +func GetOVFVAppConfigForConfigSpec( + config *vimTypes.VirtualMachineConfigInfo, + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec, + vAppData map[string]string, + vAppExData map[string]map[string]string, + templateRenderFn TemplateRenderFunc) vimTypes.BaseVmConfigSpec { + + if config.VAppConfig == nil { + // BMV: Should we really silently return here and below? + return nil + } + + vAppConfigInfo := config.VAppConfig.GetVmConfigInfo() + if vAppConfigInfo == nil { + return nil + } + + if len(vAppConfigSpec.Properties) > 0 { + vAppData = map[string]string{} + + for _, p := range vAppConfigSpec.Properties { + if p.Value.Value != nil { + vAppData[p.Key] = *p.Value.Value + } else if p.Value.From != nil { + from := p.Value.From + vAppData[p.Key] = vAppExData[from.Name][from.Key] + } + } + } + + if templateRenderFn != nil { + // If we have a templating func, apply it to whatever data we have, regardless of the source. + for k, v := range vAppData { + vAppData[k] = templateRenderFn(k, v) + } + } + + return GetMergedvAppConfigSpec(vAppData, vAppConfigInfo.Property) +} + +// GetMergedvAppConfigSpec prepares a vApp VmConfigSpec which will set the provided key/value fields. +// Only fields marked userConfigurable and pre-existing on the VM (ie. originated from the OVF Image) +// will be set, and all others will be ignored. +func GetMergedvAppConfigSpec(inProps map[string]string, vmProps []vimTypes.VAppPropertyInfo) *vimTypes.VmConfigSpec { + outProps := make([]vimTypes.VAppPropertySpec, 0) + + for _, vmProp := range vmProps { + if vmProp.UserConfigurable == nil || !*vmProp.UserConfigurable { + continue + } + + inPropValue, found := inProps[vmProp.Id] + if !found || vmProp.Value == inPropValue { + continue + } + + vmPropCopy := vmProp + vmPropCopy.Value = inPropValue + outProp := vimTypes.VAppPropertySpec{ + ArrayUpdateSpec: vimTypes.ArrayUpdateSpec{ + Operation: vimTypes.ArrayUpdateOperationEdit, + }, + Info: &vmPropCopy, + } + outProps = append(outProps, outProp) + } + + if len(outProps) == 0 { + return nil + } + + return &vimTypes.VmConfigSpec{ + Property: outProps, + // Ensure the transport is guestInfo in case the VM does not have + // a CD-ROM device required to use the ISO transport. + OvfEnvironmentTransport: []string{OvfEnvironmentTransportGuestInfo}, + } +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go new file mode 100644 index 000000000..6533b3f6b --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_vappconfig_test.go @@ -0,0 +1,267 @@ +// Copyright (c) 2021-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("VAppConfig Bootstrap", func() { + const key, value = "fooKey", "fooValue" + + var ( + configInfo *types.VirtualMachineConfigInfo + vAppConfigSpec *vmopv1.VirtualMachineBootstrapVAppConfigSpec + bsArgs vmlifecycle.BootstrapArgs + baseVMConfigSpec types.BaseVmConfigSpec + ) + + BeforeEach(func() { + configInfo = &types.VirtualMachineConfigInfo{} + configInfo.VAppConfig = &types.VmConfigInfo{ + Property: []types.VAppPropertyInfo{ + { + Id: key, + Value: "should-change", + UserConfigurable: pointer.Bool(true), + }, + }, + } + + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} + bsArgs.VAppData = make(map[string]string) + bsArgs.VAppExData = make(map[string]map[string]string) + }) + + AfterEach(func() { + vAppConfigSpec = nil + baseVMConfigSpec = nil + bsArgs = vmlifecycle.BootstrapArgs{} + }) + + Context("GetOVFVAppConfigForConfigSpec", func() { + + JustBeforeEach(func() { + baseVMConfigSpec = vmlifecycle.GetOVFVAppConfigForConfigSpec( + configInfo, + vAppConfigSpec, + bsArgs.VAppData, + bsArgs.VAppExData, + bsArgs.TemplateRenderFn) + }) + + Context("Empty input", func() { + It("No changes", func() { + Expect(baseVMConfigSpec).To(BeNil()) + }) + }) + + Context("vAppData Map", func() { + BeforeEach(func() { + bsArgs.VAppData[key] = value + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + + Context("Applies TemplateRenderFn when specified", func() { + BeforeEach(func() { + bsArgs.TemplateRenderFn = func(_, v string) string { + return strings.ToUpper(v) + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(strings.ToUpper(value))) + }) + }) + }) + + Context("vAppDataConfig Inlined Properties", func() { + BeforeEach(func() { + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{Value: pointer.String(value)}, + }, + }, + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + + Context("Applies TemplateRenderFn when specified", func() { + BeforeEach(func() { + bsArgs.TemplateRenderFn = func(_, v string) string { + return strings.ToUpper(v) + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(strings.ToUpper(value))) + }) + }) + }) + + Context("vAppDataConfig From Properties", func() { + const secretName = "my-other-secret" + + BeforeEach(func() { + vAppConfigSpec = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: key, + Value: common.ValueOrSecretKeySelector{ + From: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + Key: key, + Optional: nil, // TODO: Rethink if we really need this complexity + }, + }, + }, + }, + } + + bsArgs.VAppExData[secretName] = map[string]string{key: value} + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(value)) + }) + + Context("Applies TemplateRenderFn when specified", func() { + BeforeEach(func() { + bsArgs.TemplateRenderFn = func(_, v string) string { + return strings.ToUpper(v) + } + }) + + It("Expected VAppConfig", func() { + Expect(baseVMConfigSpec).ToNot(BeNil()) + + vmCs := baseVMConfigSpec.GetVmConfigSpec() + Expect(vmCs).ToNot(BeNil()) + Expect(vmCs.Property).To(HaveLen(1)) + Expect(vmCs.Property[0].Info).ToNot(BeNil()) + Expect(vmCs.Property[0].Info.Id).To(Equal(key)) + Expect(vmCs.Property[0].Info.Value).To(Equal(strings.ToUpper(value))) + }) + }) + }) + }) +}) + +var _ = Describe("GetMergedvAppConfigSpec", func() { + + DescribeTable("returns expected props", + func(inProps map[string]string, vmProps []types.VAppPropertyInfo, expected *types.VmConfigSpec) { + vAppConfigSpec := vmlifecycle.GetMergedvAppConfigSpec(inProps, vmProps) + if expected == nil { + Expect(vAppConfigSpec).To(BeNil()) + } else { + Expect(vAppConfigSpec.Property).To(HaveLen(len(expected.Property))) + for i := range vAppConfigSpec.Property { + Expect(vAppConfigSpec.Property[i].Info.Key).To(Equal(expected.Property[i].Info.Key)) + Expect(vAppConfigSpec.Property[i].Info.Id).To(Equal(expected.Property[i].Info.Id)) + Expect(vAppConfigSpec.Property[i].Info.Value).To(Equal(expected.Property[i].Info.Value)) + Expect(vAppConfigSpec.Property[i].ArrayUpdateSpec.Operation).To(Equal(types.ArrayUpdateOperationEdit)) + } + Expect(vAppConfigSpec.OvfEnvironmentTransport).To(HaveLen(1)) + Expect(vAppConfigSpec.OvfEnvironmentTransport[0]).To(Equal(vmlifecycle.OvfEnvironmentTransportGuestInfo)) + } + }, + Entry("return nil for absent vm and input props", + map[string]string{}, + []types.VAppPropertyInfo{}, + nil, + ), + Entry("return nil for non UserConfigurable vm props", + map[string]string{ + "one-id": "one-override-value", + "two-id": "two-override-value", + }, + []types.VAppPropertyInfo{ + {Key: 1, Id: "one-id", Value: "one-value"}, + {Key: 2, Id: "two-id", Value: "two-value", UserConfigurable: pointer.Bool(false)}, + }, + nil, + ), + Entry("return nil for UserConfigurable vm props but no input props", + map[string]string{}, + []types.VAppPropertyInfo{ + {Key: 1, Id: "one-id", Value: "one-value"}, + {Key: 2, Id: "two-id", Value: "two-value", UserConfigurable: pointer.Bool(true)}, + }, + nil, + ), + Entry("return valid vAppConfigSpec for setting mixed UserConfigurable props", + map[string]string{ + "one-id": "one-override-value", + "two-id": "two-override-value", + "three-id": "three-override-value", + }, + []types.VAppPropertyInfo{ + {Key: 1, Id: "one-id", Value: "one-value", UserConfigurable: nil}, + {Key: 2, Id: "two-id", Value: "two-value", UserConfigurable: pointer.Bool(true)}, + {Key: 3, Id: "three-id", Value: "three-value", UserConfigurable: pointer.Bool(false)}, + }, + &types.VmConfigSpec{ + Property: []types.VAppPropertySpec{ + {Info: &types.VAppPropertyInfo{Key: 2, Id: "two-id", Value: "two-override-value"}}, + }, + }, + ), + ) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go new file mode 100644 index 000000000..a6da98006 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create.go @@ -0,0 +1,41 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +// CreateArgs contains the arguments needed to create a VM. +type CreateArgs struct { + UseContentLibrary bool + ProviderItemID string + + ConfigSpec *types.VirtualMachineConfigSpec + StorageProvisioning string + FolderMoID string + ResourcePoolMoID string + HostMoID string + StorageProfileID string + DatastoreMoID string // gce2e only: used only if StorageProfileID is unset +} + +func CreateVirtualMachine( + vmCtx context.VirtualMachineContextA2, + clClient contentlibrary.Provider, + restClient *rest.Client, + finder *find.Finder, + createArgs *CreateArgs) (*types.ManagedObjectReference, error) { + + if createArgs.UseContentLibrary { + return deployFromContentLibrary(vmCtx, clClient, restClient, createArgs) + } + + return cloneVMFromInventory(vmCtx, finder, createArgs) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go new file mode 100644 index 000000000..299d0bba7 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go @@ -0,0 +1,188 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + vimtypes "github.com/vmware/govmomi/vim25/types" + "k8s.io/utils/pointer" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/placement" +) + +// CloneVMFromInventory creates a new VM by cloning the source VM. This is not reachable/used +// in production because we only really support deploying an OVF via content library. +// Maybe someday we'll use clone to speed up VM deployment so keep this code around and unit tested. +func cloneVMFromInventory( + vmCtx context.VirtualMachineContextA2, + finder *find.Finder, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + srcVMName := createArgs.ProviderItemID // AKA: vmCtx.VM.Spec.ImageName + + srcVM, err := finder.VirtualMachine(vmCtx, srcVMName) + if err != nil { + return nil, errors.Wrapf(err, "failed to find clone source VM: %s", srcVMName) + } + + cloneSpec, err := createCloneSpec(vmCtx, createArgs, srcVM) + if err != nil { + return nil, errors.Wrap(err, "failed to create CloneSpec") + } + + // We always set cloneSpec.Location.Folder so use that to get the parent folder object. + folder := object.NewFolder(srcVM.Client(), *cloneSpec.Location.Folder) + + cloneTask, err := srcVM.Clone(vmCtx, folder, cloneSpec.Config.Name, *cloneSpec) + if err != nil { + return nil, err + } + + result, err := cloneTask.WaitForResult(vmCtx, nil) + if err != nil { + return nil, errors.Wrapf(err, "clone VM task failed") + } + + ref := result.Result.(vimtypes.ManagedObjectReference) + return &ref, nil +} + +func createCloneSpec( + vmCtx context.VirtualMachineContextA2, + createArgs *CreateArgs, + srcVM *object.VirtualMachine) (*vimtypes.VirtualMachineCloneSpec, error) { + + cloneSpec := &vimtypes.VirtualMachineCloneSpec{ + Config: createArgs.ConfigSpec, + Memory: pointer.Bool(false), // No full memory clones. + } + + virtualDevices, err := srcVM.Device(vmCtx) + if err != nil { + return nil, fmt.Errorf("failed to get clone source VM devices: %w", err) + } + + virtualDisks := virtualDevices.SelectByType((*vimtypes.VirtualDisk)(nil)) + + for _, deviceChange := range resizeBootDiskDeviceChange(vmCtx, virtualDisks) { + if deviceChange.GetVirtualDeviceConfigSpec().Operation == vimtypes.VirtualDeviceConfigSpecOperationEdit { + cloneSpec.Location.DeviceChange = append(cloneSpec.Location.DeviceChange, deviceChange) + } else { + cloneSpec.Config.DeviceChange = append(cloneSpec.Config.DeviceChange, deviceChange) + } + } + + if createArgs.StorageProfileID != "" { + cloneSpec.Location.Profile = []vimtypes.BaseVirtualMachineProfileSpec{ + &vimtypes.VirtualMachineDefinedProfileSpec{ProfileId: createArgs.StorageProfileID}, + } + } else { + // BMV: Used to compute placement? Otherwise, always overwritten later. + cloneSpec.Location.Datastore = &vimtypes.ManagedObjectReference{ + Type: "Datastore", + Value: createArgs.DatastoreMoID, + } + } + + cloneSpec.Location.Pool = &vimtypes.ManagedObjectReference{ + Type: "ResourcePool", + Value: createArgs.ResourcePoolMoID, + } + cloneSpec.Location.Folder = &vimtypes.ManagedObjectReference{ + Type: "Folder", + Value: createArgs.FolderMoID, + } + + rpOwner, err := object.NewResourcePool(srcVM.Client(), *cloneSpec.Location.Pool).Owner(vmCtx) + if err != nil { + return nil, err + } + + cluster, ok := rpOwner.(*object.ClusterComputeResource) + if !ok { + return nil, fmt.Errorf("owner of the ResourcePool is not a cluster but %T", rpOwner) + } + + relocateSpec, err := placement.CloneVMRelocateSpec(vmCtx, cluster, srcVM.Reference(), cloneSpec) + if err != nil { + return nil, err + } + + cloneSpec.Location.Host = relocateSpec.Host + cloneSpec.Location.Datastore = relocateSpec.Datastore + cloneSpec.Location.Disk = cloneVMDiskLocators(virtualDisks, createArgs, cloneSpec.Location) + + return cloneSpec, nil +} + +func cloneVMDiskLocators( + disks object.VirtualDeviceList, + createArgs *CreateArgs, + location vimtypes.VirtualMachineRelocateSpec) []vimtypes.VirtualMachineRelocateSpecDiskLocator { + + diskLocators := make([]vimtypes.VirtualMachineRelocateSpecDiskLocator, 0, len(disks)) + + for _, disk := range disks { + locator := vimtypes.VirtualMachineRelocateSpecDiskLocator{ + DiskId: disk.GetVirtualDevice().Key, + Datastore: *location.Datastore, + Profile: location.Profile, + // TODO: Check if policy is encrypted and use correct DiskMoveType + DiskMoveType: string(vimtypes.VirtualMachineRelocateDiskMoveOptionsMoveChildMostDiskBacking), + } + + if backing, ok := disk.(*vimtypes.VirtualDisk).Backing.(*vimtypes.VirtualDiskFlatVer2BackingInfo); ok { + switch createArgs.StorageProvisioning { + case string(vimtypes.OvfCreateImportSpecParamsDiskProvisioningTypeThin): + backing.ThinProvisioned = pointer.Bool(true) + case string(vimtypes.OvfCreateImportSpecParamsDiskProvisioningTypeThick): + backing.ThinProvisioned = pointer.Bool(false) + case string(vimtypes.OvfCreateImportSpecParamsDiskProvisioningTypeEagerZeroedThick): + backing.EagerlyScrub = pointer.Bool(true) + } + locator.DiskBackingInfo = backing + } + + diskLocators = append(diskLocators, locator) + } + + return diskLocators +} + +func resizeBootDiskDeviceChange( + vmCtx context.VirtualMachineContextA2, + virtualDisks object.VirtualDeviceList) []vimtypes.BaseVirtualDeviceConfigSpec { + + capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity + if capacity.IsZero() { + return nil + } + + // Assume the first virtual disk - if any - is the boot disk. + var deviceChanges []vimtypes.BaseVirtualDeviceConfigSpec + for _, vmDevice := range virtualDisks { + vmDisk, ok := vmDevice.(*vimtypes.VirtualDisk) + if !ok { + continue + } + + // Maybe don't allow shrink? + if vmDisk.CapacityInBytes != capacity.Value() { + vmDisk.CapacityInBytes = capacity.Value() + deviceChanges = append(deviceChanges, &vimtypes.VirtualDeviceConfigSpec{ + Operation: vimtypes.VirtualDeviceConfigSpecOperationEdit, + Device: vmDisk, + }) + } + + break + } + + return deviceChanges +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go new file mode 100644 index 000000000..e0edefb55 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_contentlibrary.go @@ -0,0 +1,105 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + "encoding/base64" + "fmt" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vapi/rest" + "github.com/vmware/govmomi/vapi/vcenter" + vimtypes "github.com/vmware/govmomi/vim25/types" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" +) + +func deployOVF( + vmCtx context.VirtualMachineContextA2, + restClient *rest.Client, + item *library.Item, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + deploymentSpec := vcenter.DeploymentSpec{ + Name: vmCtx.VM.Name, + StorageProfileID: createArgs.StorageProfileID, + StorageProvisioning: createArgs.StorageProvisioning, + AcceptAllEULA: true, + } + + if deploymentSpec.StorageProfileID == "" { + // Without a storage profile, fall back to the datastore. + deploymentSpec.DefaultDatastoreID = createArgs.DatastoreMoID + } + + if lib.IsVMClassAsConfigFSSDaynDateEnabled() && createArgs.ConfigSpec != nil { + configSpecXML, err := util.MarshalConfigSpecToXML(createArgs.ConfigSpec) + if err != nil { + return nil, fmt.Errorf("failed to marshal ConfigSpec to XML: %w", err) + } + + deploymentSpec.VmConfigSpec = &vcenter.VmConfigSpec{ + Provider: constants.ConfigSpecProviderXML, + XML: base64.StdEncoding.EncodeToString(configSpecXML), + } + } + + deploy := vcenter.Deploy{ + DeploymentSpec: deploymentSpec, + Target: vcenter.Target{ + ResourcePoolID: createArgs.ResourcePoolMoID, + FolderID: createArgs.FolderMoID, + HostID: createArgs.HostMoID, + }, + } + + vmCtx.Logger.Info("Deploying OVF Library Item", "itemID", item.ID, "itemName", item.Name, "deploy", deploy) + + return vcenter.NewManager(restClient).DeployLibraryItem(vmCtx, item.ID, deploy) +} + +func deployVMTX( + vmCtx context.VirtualMachineContextA2, + restClient *rest.Client, + item *library.Item, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + // Not yet supported. This item type needs to be deployed via DeployTemplateLibraryItem(), + // which doesn't take a ConfigSpec so it is a heavy lift. + // TODO: We should catch this earlier to avoid a bunch of wasted work. + + _ = vmCtx + _ = restClient + _ = createArgs + + return nil, fmt.Errorf("creating VM from VMTX content library type is not supported: %s", item.Name) +} + +func deployFromContentLibrary( + vmCtx context.VirtualMachineContextA2, + clClient contentlibrary.Provider, + restClient *rest.Client, + createArgs *CreateArgs) (*vimtypes.ManagedObjectReference, error) { + + // This call is needed to get the item type. We could avoid going to CL here, and + // instead get the item type via the {Cluster}ContentLibrary CR for the image. + item, err := clClient.GetLibraryItemID(vmCtx, createArgs.ProviderItemID) + if err != nil { + return nil, err + } + + switch item.Type { + case library.ItemTypeOVF: + return deployOVF(vmCtx, restClient, item, createArgs) + case library.ItemTypeVMTX: + return deployVMTX(vmCtx, restClient, item, createArgs) + default: + return nil, errors.Errorf("item %s not a supported type: %s", item.Name, item.Type) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go new file mode 100644 index 000000000..62bf35ba4 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go @@ -0,0 +1,345 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle + +import ( + goctx "context" + "fmt" + "net" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" +) + +var ( + // The minimum properties needed to be retrieved in order to populate the Status. Callers may + // provide a MO with more. This often saves us a second round trip in the common steady state. + vmStatusPropertiesSelector = []string{"config.changeTrackingEnabled", "guest", "summary"} +) + +func UpdateStatus( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client, + vcVM *object.VirtualMachine, + vmMO *mo.VirtualMachine) error { + + vm := vmCtx.VM + + // This is implicitly true: ensure the condition is set since it is how we determine the old v1a1 Phase. + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionCreated) + // TODO: Might set other "prereq" conditions too for version conversion but we'd have to fib a little. + + if vm.Status.Image == nil { + // If unset, we don't know if this was a cluster or namespace scoped image at the time. + vm.Status.Image = &common.LocalObjectRef{Name: vm.Spec.ImageName} + } + if vm.Status.Class == nil { + // We can most likely just assume the other fields of the LocalObjectRef but only fill + // in the Name for now. Our handling of this field will be more complicated once we really + // support class changes and reconfiguring the VM the fly in response. + vm.Status.Class = &common.LocalObjectRef{Name: vm.Spec.ClassName} + } + + if vmMO == nil { + // In the common case, our caller will have already gotten the MO properties in order to determine + // if it had any reconciliation to do, and there was nothing to do since the VM is in the steady + // state so that MO is still entirely valid here. + // NOTE: The properties must have been retrieved with at least vmStatusPropertiesSelector. + vmMO = &mo.VirtualMachine{} + if err := vcVM.Properties(vmCtx, vcVM.Reference(), vmStatusPropertiesSelector, vmMO); err != nil { + // Leave the current Status unchanged for now. + return fmt.Errorf("failed to get VM properties for status update: %w", err) + } + } + + var errs []error + var err error + summary := vmMO.Summary + + vm.Status.PowerState = convertPowerState(summary.Runtime.PowerState) + vm.Status.UniqueID = vcVM.Reference().Value + vm.Status.BiosUUID = summary.Config.Uuid + vm.Status.InstanceUUID = summary.Config.InstanceUuid + vm.Status.Network = getGuestNetworkStatus(vmMO.Guest) + + vm.Status.Host, err = getRuntimeHostHostname(vmCtx, vcVM, summary.Runtime.Host) + if err != nil { + errs = append(errs, err) + } + + MarkVMToolsRunningStatusCondition(vmCtx.VM, vmMO.Guest) + MarkCustomizationInfoCondition(vmCtx.VM, vmMO.Guest) + + if config := vmMO.Config; config != nil { + vm.Status.ChangeBlockTracking = config.ChangeTrackingEnabled + } else { + vm.Status.ChangeBlockTracking = nil + } + + if lib.IsWcpFaultDomainsFSSEnabled() { + zoneName := vm.Labels[topology.KubernetesTopologyZoneLabelKey] + if zoneName == "" { + cluster, err := virtualmachine.GetVMClusterComputeResource(vmCtx, vcVM) + if err != nil { + errs = append(errs, err) + } else { + zoneName, err = topology.LookupZoneForClusterMoID(vmCtx, k8sClient, cluster.Reference().Value) + if err != nil { + errs = append(errs, err) + } else { + if vm.Labels == nil { + vm.Labels = map[string]string{} + } + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + } + } + } + + if zoneName != "" { + vm.Status.Zone = zoneName + } + } + + return k8serrors.NewAggregate(errs) +} + +func getRuntimeHostHostname( + ctx goctx.Context, + vcVM *object.VirtualMachine, + host *types.ManagedObjectReference) (string, error) { + + if host != nil { + return object.NewHostSystem(vcVM.Client(), *host).ObjectName(ctx) + } + return "", nil +} + +func getGuestNetworkStatus(guestInfo *types.GuestInfo) *vmopv1.VirtualMachineNetworkStatus { + if guestInfo == nil { + return nil + } + + status := &vmopv1.VirtualMachineNetworkStatus{} + + if ipAddr := guestInfo.IpAddress; ipAddr != "" { + // TODO: Filter out local addresses. + if net.ParseIP(ipAddr).To4() != nil { + status.PrimaryIP4 = ipAddr + } else { + status.PrimaryIP6 = ipAddr + } + } + + if len(guestInfo.Net) > 0 { + status.Interfaces = make([]vmopv1.VirtualMachineNetworkInterfaceStatus, 0, len(guestInfo.Net)) + for i := range guestInfo.Net { + status.Interfaces = append(status.Interfaces, guestNicInfoToInterfaceStatus(i, &guestInfo.Net[i])) + } + } + + if len(guestInfo.IpStack) > 0 { + status.VirtualMachineNetworkIPStackStatus = guestIPStackInfoToIPStackStatus(&guestInfo.IpStack[0]) + } + + return status +} + +func guestNicInfoToInterfaceStatus(idx int, guestNicInfo *types.GuestNicInfo) vmopv1.VirtualMachineNetworkInterfaceStatus { + status := vmopv1.VirtualMachineNetworkInterfaceStatus{} + + // TODO: What name exactly? The CRD name may be the most useful here but hard to line that up. + // BMV: DeviceConfigId will be -1 for our pseudo-y interfaces. Most likely want to just skip those devices. + status.Name = fmt.Sprintf("nic-%d-%d", idx, guestNicInfo.DeviceConfigId) + status.IP.MACAddr = guestNicInfo.MacAddress + + if guestIPConfig := guestNicInfo.IpConfig; guestIPConfig != nil { + ip := &status.IP + + ip.AutoConfigurationEnabled = guestIPConfig.AutoConfigurationEnabled + ip.Addresses = convertNetIPConfigInfoIPAddresses(guestIPConfig.IpAddress) + + if guestIPConfig.Dhcp != nil { + ip.DHCP = convertNetDhcpConfigInfo(guestIPConfig.Dhcp) + } + } + + if dnsConfig := guestNicInfo.DnsConfig; dnsConfig != nil { + status.DNS = convertNetDNSConfigInfo(dnsConfig) + } + + return status +} + +func guestIPStackInfoToIPStackStatus(guestIPStack *types.GuestStackInfo) vmopv1.VirtualMachineNetworkIPStackStatus { + status := vmopv1.VirtualMachineNetworkIPStackStatus{} + + if dhcpConfig := guestIPStack.DhcpConfig; dhcpConfig != nil { + status.DHCP = convertNetDhcpConfigInfo(dhcpConfig) + } + + if dnsConfig := guestIPStack.DnsConfig; dnsConfig != nil { + status.DNS = convertNetDNSConfigInfo(dnsConfig) + } + + if ipRouteConfig := guestIPStack.IpRouteConfig; ipRouteConfig != nil { + status.IPRoutes = convertNetIPRouteConfigInfo(ipRouteConfig) + } + + status.KernelConfig = convertKeyValueSlice(guestIPStack.IpStackConfig) + + return status +} + +func convertPowerState(powerState types.VirtualMachinePowerState) vmopv1.VirtualMachinePowerState { + switch powerState { + case types.VirtualMachinePowerStatePoweredOff: + return vmopv1.VirtualMachinePowerStateOff + case types.VirtualMachinePowerStatePoweredOn: + return vmopv1.VirtualMachinePowerStateOn + case types.VirtualMachinePowerStateSuspended: + return vmopv1.VirtualMachinePowerStateSuspended + } + return "" +} + +func convertNetIPConfigInfoIPAddresses(ipAddresses []types.NetIpConfigInfoIpAddress) []vmopv1.VirtualMachineNetworkInterfaceIPAddrStatus { + if len(ipAddresses) == 0 { + return nil + } + + out := make([]vmopv1.VirtualMachineNetworkInterfaceIPAddrStatus, 0, len(ipAddresses)) + for _, guestIPAddr := range ipAddresses { + ipAddrStatus := vmopv1.VirtualMachineNetworkInterfaceIPAddrStatus{ + Address: guestIPAddr.IpAddress, + Origin: guestIPAddr.Origin, + State: guestIPAddr.State, + } + if guestIPAddr.Lifetime != nil { + ipAddrStatus.Lifetime = metav1.NewTime(*guestIPAddr.Lifetime) + } + + out = append(out, ipAddrStatus) + } + return out +} + +func convertNetDNSConfigInfo(dnsConfig *types.NetDnsConfigInfo) vmopv1.VirtualMachineNetworkDNSStatus { + return vmopv1.VirtualMachineNetworkDNSStatus{ + DHCP: dnsConfig.Dhcp, + DomainName: dnsConfig.DomainName, + HostName: dnsConfig.HostName, + Nameservers: dnsConfig.IpAddress, + SearchDomains: dnsConfig.SearchDomain, + } +} + +func convertNetDhcpConfigInfo(dhcpConfig *types.NetDhcpConfigInfo) vmopv1.VirtualMachineNetworkDHCPStatus { + status := vmopv1.VirtualMachineNetworkDHCPStatus{} + + if ipv4 := dhcpConfig.Ipv4; ipv4 != nil { + status.IP4.Enabled = ipv4.Enable + status.IP4.Config = convertKeyValueSlice(ipv4.Config) + } + + if ipv6 := dhcpConfig.Ipv6; ipv6 != nil { + status.IP6.Enabled = ipv6.Enable + status.IP6.Config = convertKeyValueSlice(ipv6.Config) + } + + return status +} + +func convertNetIPRouteConfigInfo(routeConfig *types.NetIpRouteConfigInfo) []vmopv1.VirtualMachineNetworkIPRouteStatus { + if len(routeConfig.IpRoute) == 0 { + return nil + } + + // TODO: Prob only want to show default routes. Will be very verbose on TKG nodes. + out := make([]vmopv1.VirtualMachineNetworkIPRouteStatus, 0, len(routeConfig.IpRoute)) + for _, ipRoute := range routeConfig.IpRoute { + out = append(out, vmopv1.VirtualMachineNetworkIPRouteStatus{ + Gateway: vmopv1.VirtualMachineNetworkIPRouteGatewayStatus{ + Device: ipRoute.Gateway.Device, + Address: ipRoute.Gateway.IpAddress, + }, + NetworkAddress: fmt.Sprintf("%s/%d", ipRoute.Network, ipRoute.PrefixLength), + }) + } + return out +} + +func convertKeyValueSlice(s []types.KeyValue) []common.KeyValuePair { + if len(s) == 0 { + return nil + } + + out := make([]common.KeyValuePair, 0, len(s)) + for i := range s { + out = append(out, common.KeyValuePair{Key: s[i].Key, Value: s[i].Value}) + } + return out +} + +func MarkVMToolsRunningStatusCondition( + vm *vmopv1.VirtualMachine, + guestInfo *types.GuestInfo) { + + if guestInfo == nil || guestInfo.ToolsRunningStatus == "" { + conditions.MarkUnknown(vm, vmopv1.VirtualMachineToolsCondition, "NoGuestInfo", "") + return + } + + switch guestInfo.ToolsRunningStatus { + case string(types.VirtualMachineToolsRunningStatusGuestToolsNotRunning): + msg := "VMware Tools is not running" + conditions.MarkFalse(vm, vmopv1.VirtualMachineToolsCondition, vmopv1.VirtualMachineToolsNotRunningReason, msg) + case string(types.VirtualMachineToolsRunningStatusGuestToolsRunning), string(types.VirtualMachineToolsRunningStatusGuestToolsExecutingScripts): + conditions.MarkTrue(vm, vmopv1.VirtualMachineToolsCondition) + default: + msg := "Unexpected VMware Tools running status" + conditions.MarkUnknown(vm, vmopv1.VirtualMachineToolsCondition, "Unknown", msg) + } +} + +func MarkCustomizationInfoCondition(vm *vmopv1.VirtualMachine, guestInfo *types.GuestInfo) { + if guestInfo == nil || guestInfo.CustomizationInfo == nil { + conditions.MarkUnknown(vm, vmopv1.GuestCustomizationCondition, "NoGuestInfo", "") + return + } + + switch guestInfo.CustomizationInfo.CustomizationStatus { + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_IDLE), "": + conditions.MarkTrue(vm, vmopv1.GuestCustomizationCondition) + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_PENDING): + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationPendingReason, "") + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_RUNNING): + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationRunningReason, "") + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_SUCCEEDED): + conditions.MarkTrue(vm, vmopv1.GuestCustomizationCondition) + case string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_FAILED): + errorMsg := guestInfo.CustomizationInfo.ErrorMsg + if errorMsg == "" { + errorMsg = "vSphere VM Customization failed due to an unknown error." + } + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationFailedReason, errorMsg) + default: + errorMsg := guestInfo.CustomizationInfo.ErrorMsg + if errorMsg == "" { + errorMsg = "Unexpected VM Customization status" + } + conditions.MarkFalse(vm, vmopv1.GuestCustomizationCondition, "Unknown", errorMsg) + } +} diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go new file mode 100644 index 000000000..3d130cdc9 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go @@ -0,0 +1,211 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +var _ = Describe("VirtualMachineTools Status to VM Status Condition", func() { + Context("markVMToolsRunningStatusCondition", func() { + var ( + vm *vmopv1.VirtualMachine + guestInfo *types.GuestInfo + ) + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{} + guestInfo = &types.GuestInfo{ + ToolsRunningStatus: "", + } + }) + + JustBeforeEach(func() { + vmlifecycle.MarkVMToolsRunningStatusCondition(vm, guestInfo) + }) + + Context("guestInfo is nil", func() { + BeforeEach(func() { + guestInfo = nil + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.VirtualMachineToolsCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("ToolsRunningStatus is empty", func() { + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.VirtualMachineToolsCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("vmtools is not running", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = string(types.VirtualMachineToolsRunningStatusGuestToolsNotRunning) + }) + It("sets condition to false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.VirtualMachineToolsCondition, vmopv1.VirtualMachineToolsNotRunningReason, "VMware Tools is not running"), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("vmtools is running", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = string(types.VirtualMachineToolsRunningStatusGuestToolsRunning) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.VirtualMachineToolsCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("vmtools is starting", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = string(types.VirtualMachineToolsRunningStatusGuestToolsExecutingScripts) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.VirtualMachineToolsCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("Unexpected vmtools running status", func() { + BeforeEach(func() { + guestInfo.ToolsRunningStatus = "blah" + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.VirtualMachineToolsCondition, "Unknown", "Unexpected VMware Tools running status"), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + }) +}) + +var _ = Describe("VSphere Customization Status to VM Status Condition", func() { + Context("markCustomizationInfoCondition", func() { + var ( + vm *vmopv1.VirtualMachine + guestInfo *types.GuestInfo + ) + + BeforeEach(func() { + vm = &vmopv1.VirtualMachine{} + guestInfo = &types.GuestInfo{ + CustomizationInfo: &types.GuestInfoCustomizationInfo{}, + } + }) + + JustBeforeEach(func() { + vmlifecycle.MarkCustomizationInfoCondition(vm, guestInfo) + }) + + Context("guestInfo unset", func() { + BeforeEach(func() { + guestInfo = nil + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.GuestCustomizationCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo unset", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo = nil + }) + It("sets condition unknown", func() { + expectedConditions := []metav1.Condition{ + *conditions.UnknownCondition(vmopv1.GuestCustomizationCondition, "NoGuestInfo", ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo idle", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_IDLE) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.GuestCustomizationCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo pending", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_PENDING) + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationPendingReason, ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo running", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_RUNNING) + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationRunningReason, ""), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo succeeded", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_SUCCEEDED) + }) + It("sets condition true", func() { + expectedConditions := []metav1.Condition{ + *conditions.TrueCondition(vmopv1.GuestCustomizationCondition), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo failed", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = string(types.GuestInfoCustomizationStatusTOOLSDEPLOYPKG_FAILED) + guestInfo.CustomizationInfo.ErrorMsg = "some error message" + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, vmopv1.GuestCustomizationFailedReason, guestInfo.CustomizationInfo.ErrorMsg), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + Context("customizationInfo invalid", func() { + BeforeEach(func() { + guestInfo.CustomizationInfo.CustomizationStatus = "asdf" + guestInfo.CustomizationInfo.ErrorMsg = "some error message" + }) + It("sets condition false", func() { + expectedConditions := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.GuestCustomizationCondition, "Unknown", guestInfo.CustomizationInfo.ErrorMsg), + } + Expect(vm.Status.Conditions).To(conditions.MatchConditions(expectedConditions)) + }) + }) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go new file mode 100644 index 000000000..461db366f --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/vmlifecycle_suite_test.go @@ -0,0 +1,22 @@ +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vmlifecycle_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() +var _ = BeforeSuite(suite.BeforeSuite) +var _ = AfterSuite(suite.AfterSuite) + +func TestVMLifecycle(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "vSphere Provider VM Lifecycle Suite") +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider.go b/pkg/vmprovider/providers/vsphere2/vmprovider.go new file mode 100644 index 000000000..161b24262 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider.go @@ -0,0 +1,428 @@ +// Copyright (c) 2018-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + goctx "context" + "encoding/json" + "fmt" + "math/rand" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/ovf" + "github.com/vmware/govmomi/task" + "github.com/vmware/govmomi/vapi/library" + "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + ctrlruntime "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/record" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + vcconfig "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" +) + +const ( + VsphereVMProviderName = "vsphere" + + // taskHistoryCollectorPageSize represents the max count to read from task manager in one iteration. + taskHistoryCollectorPageSize = 10 + + ovfCacheMaxItem = 100 + ovfCacheItemExpiration = 30 * time.Minute + ovfCacheExpirationCheckInterval = 5 * time.Minute +) + +var log = logf.Log.WithName(VsphereVMProviderName) + +type VersionedOVFEnvelope struct { + OvfEnvelope *ovf.Envelope + ContentVersion string +} + +type vSphereVMProvider struct { + k8sClient ctrlruntime.Client + eventRecorder record.Recorder + globalExtraConfig map[string]string + minCPUFreq uint64 + ovfCache *util.Cache[VersionedOVFEnvelope] + ovfCacheLockPool *util.LockPool[string, *sync.RWMutex] + + vcClientLock sync.Mutex + vcClient *vcclient.Client +} + +func NewVSphereVMProviderFromClient( + client ctrlruntime.Client, + recorder record.Recorder) vmprovider.VirtualMachineProviderInterfaceA2 { + + ovfCache, ovfLockPool := InitOvfCacheAndLockPool( + ovfCacheItemExpiration, ovfCacheExpirationCheckInterval, ovfCacheMaxItem) + + return &vSphereVMProvider{ + k8sClient: client, + eventRecorder: recorder, + globalExtraConfig: getExtraConfig(), + ovfCache: ovfCache, + ovfCacheLockPool: ovfLockPool, + } +} + +// InitOvfCacheAndLockPool initializes the ovf cache and lock pool that are used +// to cache the ovf envelope and lock the ovf envelope when it is being downloaded. +func InitOvfCacheAndLockPool(expireAfter, checkExpireInterval time.Duration, maxItems int) ( + *util.Cache[VersionedOVFEnvelope], *util.LockPool[string, *sync.RWMutex]) { + ovfCache := util.NewCache[VersionedOVFEnvelope](expireAfter, checkExpireInterval, maxItems) + ovfLockPool := &util.LockPool[string, *sync.RWMutex]{} + + // Clean up the lock pool when the ovf cache item expires. + expiredChan := ovfCache.ExpiredChan() + go func() { + for k := range expiredChan { + l := ovfLockPool.Get(k) + // This could still delete an in-use lock if it's retrieved from the pool but not locked yet. + // If it's already locked, this will wait until it's unlocked to delete it from the pool. + l.Lock() + ovfLockPool.Delete(k) + l.Unlock() + } + }() + + return ovfCache, ovfLockPool +} + +func getExtraConfig() map[string]string { + ec := map[string]string{ + constants.EnableDiskUUIDExtraConfigKey: constants.ExtraConfigTrue, + constants.GOSCIgnoreToolsCheckExtraConfigKey: constants.ExtraConfigTrue, + } + + if jsonEC := os.Getenv("JSON_EXTRA_CONFIG"); jsonEC != "" { + extraConfig := make(map[string]string) + + if err := json.Unmarshal([]byte(jsonEC), &extraConfig); err != nil { + // This is only set in testing so make errors fatal. + panic(fmt.Sprintf("invalid JSON_EXTRA_CONFIG envvar: %q %v", jsonEC, err)) + } + + for k, v := range extraConfig { + ec[k] = v + } + } + + return ec +} + +func (vs *vSphereVMProvider) getVcClient(ctx goctx.Context) (*vcclient.Client, error) { + vs.vcClientLock.Lock() + defer vs.vcClientLock.Unlock() + + if vs.vcClient != nil { + return vs.vcClient, nil + } + + config, err := vcconfig.GetProviderConfig(ctx, vs.k8sClient) + if err != nil { + return nil, err + } + + vcClient, err := vcclient.NewClient(ctx, config) + if err != nil { + return nil, err + } + + vs.vcClient = vcClient + return vcClient, nil +} + +func (vs *vSphereVMProvider) UpdateVcPNID(ctx goctx.Context, vcPNID, vcPort string) error { + updated, err := vcconfig.UpdateVcInConfigMap(ctx, vs.k8sClient, vcPNID, vcPort) + if err != nil || !updated { + return err + } + + // Our controller-runtime client does not cache ConfigMaps & Secrets, so the next time + // getVcClient() is called, it will fetch newly updated CM. + vs.clearAndLogoutVcClient(ctx) + return nil +} + +func (vs *vSphereVMProvider) ResetVcClient(ctx goctx.Context) { + vs.clearAndLogoutVcClient(ctx) +} + +func (vs *vSphereVMProvider) clearAndLogoutVcClient(ctx goctx.Context) { + vs.vcClientLock.Lock() + vcClient := vs.vcClient + vs.vcClient = nil + vs.vcClientLock.Unlock() + + if vcClient != nil { + vcClient.Logout(ctx) + } +} + +// SyncVirtualMachineImage syncs the vmi object with the OVF Envelope retrieved from the cli object. +func (vs *vSphereVMProvider) SyncVirtualMachineImage(ctx goctx.Context, cli, vmi ctrlruntime.Object) error { + var itemID, contentVersion string + switch cli := cli.(type) { + case *imgregv1a1.ContentLibraryItem: + itemID = string(cli.Spec.UUID) + contentVersion = cli.Status.ContentVersion + case *imgregv1a1.ClusterContentLibraryItem: + itemID = string(cli.Spec.UUID) + contentVersion = cli.Status.ContentVersion + default: + return errors.Errorf("unexpected content library item type %T", cli) + } + + ovfEnvelope, err := vs.getOvfEnvelope(ctx, itemID, contentVersion) + if err != nil { + return err + } + + logger := log.V(4).WithValues("vmiName", vmi.GetName(), "cliName", cli.GetName()) + if ovfEnvelope == nil { + logger.Error(nil, "skip syncing VMI as corresponding OVF envelope is nil") + return nil + } + + contentlibrary.UpdateVmiWithOvfEnvelope(vmi, *ovfEnvelope) + return nil +} + +// getOvfEnvelope gets the OVF envelope from the cache if it exists and matches version. +// If not, it downloads the OVF envelope from vCenter and stores it in the cache. +func (vs *vSphereVMProvider) getOvfEnvelope( + ctx goctx.Context, itemID, contentVersion string) (*ovf.Envelope, error) { + logger := log.V(4).WithValues("itemID", itemID, "contentVersion", contentVersion) + + // Lock the current item to prevent concurrent downloads of the same OVF. + // This is done before the get from cache below to prevent stale result. + curItemLock := vs.ovfCacheLockPool.Get(itemID) + curItemLock.Lock() + defer curItemLock.Unlock() + + isHitFunc := func(cacheItem VersionedOVFEnvelope) bool { + return cacheItem.ContentVersion == contentVersion + } + cacheItem, found := vs.ovfCache.Get(itemID, isHitFunc) + if found { + logger.Info("Cache item hit, using cached OVF") + } else { + logger.Info("Cache item miss, downloading OVF from vCenter") + client, err := vs.getVcClient(ctx) + if err != nil { + return nil, err + } + + ovfEnvelope, err := client.ContentLibClient().RetrieveOvfEnvelopeByLibraryItemID(ctx, itemID) + if err != nil || ovfEnvelope == nil { + return nil, err + } + + cacheItem = VersionedOVFEnvelope{ + ContentVersion: contentVersion, + OvfEnvelope: ovfEnvelope, + } + putResult := vs.ovfCache.Put(itemID, cacheItem) + logger.Info("Cache item put", "itemID", itemID, "putResult", putResult) + } + + return cacheItem.OvfEnvelope, nil +} + +// GetItemFromLibraryByName get the library item from specified content library by its name. +// Do not return error if the item doesn't exist in the content library. +func (vs *vSphereVMProvider) GetItemFromLibraryByName(ctx goctx.Context, + contentLibrary, itemName string) (*library.Item, error) { + log.V(4).Info("Get item from ContentLibrary", + "UUID", contentLibrary, "item name", itemName) + + client, err := vs.getVcClient(ctx) + if err != nil { + return nil, err + } + + return client.ContentLibClient().GetLibraryItem(ctx, contentLibrary, itemName, false) +} + +func (vs *vSphereVMProvider) UpdateContentLibraryItem(ctx goctx.Context, itemID, newName string, newDescription *string) error { + log.V(4).Info("Update Content Library Item", "itemID", itemID) + + client, err := vs.getVcClient(ctx) + if err != nil { + return err + } + + return client.ContentLibClient().UpdateLibraryItem(ctx, itemID, newName, newDescription) +} + +func (vs *vSphereVMProvider) getOpID(vm *vmopv1.VirtualMachine, operation string) string { + const charset = "0123456789abcdef" + + id := make([]byte, 8) + for i := range id { + idx := rand.Intn(len(charset)) //nolint:gosec + id[i] = charset[idx] + } + + return strings.Join([]string{"vmoperator", vm.Name, operation, string(id)}, "-") +} + +func (vs *vSphereVMProvider) getVM( + vmCtx context.VirtualMachineContextA2, + client *vcclient.Client, + notFoundReturnErr bool) (*object.VirtualMachine, error) { + + vcVM, err := vcenter.GetVirtualMachine(vmCtx, vs.k8sClient, client.VimClient(), client.Datacenter(), client.Finder()) + if err != nil { + return nil, err + } + + if vcVM == nil && notFoundReturnErr { + return nil, fmt.Errorf("VirtualMachine %q was not found on VC", vmCtx.VM.Name) + } + + return vcVM, nil +} + +func (vs *vSphereVMProvider) getOrComputeCPUMinFrequency(ctx goctx.Context) (uint64, error) { + minFreq := atomic.LoadUint64(&vs.minCPUFreq) + if minFreq == 0 { + // The infra controller hasn't finished ComputeCPUMinFrequency() yet, so try to + // compute that value now. + var err error + minFreq, err = vs.computeCPUMinFrequency(ctx) + if err != nil { + // minFreq may be non-zero in case of partial success. + return minFreq, err + } + + // Update value if not updated already. + atomic.CompareAndSwapUint64(&vs.minCPUFreq, 0, minFreq) + } + + return minFreq, nil +} + +func (vs *vSphereVMProvider) ComputeCPUMinFrequency(ctx goctx.Context) error { + minFreq, err := vs.computeCPUMinFrequency(ctx) + if err != nil { + // Might have a partial success (non-zero freq): store that if we haven't updated + // the min freq yet, and let the controller retry. This whole min CPU freq thing + // is kind of unfortunate & busted. + atomic.CompareAndSwapUint64(&vs.minCPUFreq, 0, minFreq) + return err + } + + atomic.StoreUint64(&vs.minCPUFreq, minFreq) + return nil +} + +func (vs *vSphereVMProvider) computeCPUMinFrequency(ctx goctx.Context) (uint64, error) { + // Get all the availability zones in order to calculate the minimum + // CPU frequencies for each of the zones' vSphere clusters. + availabilityZones, err := topology.GetAvailabilityZones(ctx, vs.k8sClient) + if err != nil { + return 0, err + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return 0, err + } + + if !lib.IsWcpFaultDomainsFSSEnabled() { + ccr, err := vcenter.GetResourcePoolOwnerMoRef(ctx, client.VimClient(), client.Config().ResourcePool) + if err != nil { + return 0, err + } + + // Only expect 1 AZ in this case. + for i := range availabilityZones { + availabilityZones[i].Spec.ClusterComputeResourceMoIDs = []string{ccr.Value} + } + } + + var errs []error + + var minFreq uint64 + for _, az := range availabilityZones { + moIDs := az.Spec.ClusterComputeResourceMoIDs + if len(moIDs) == 0 { + moIDs = []string{az.Spec.ClusterComputeResourceMoId} // HA TEMP + } + + for _, moID := range moIDs { + ccr := object.NewClusterComputeResource(client.VimClient(), + types.ManagedObjectReference{Type: "ClusterComputeResource", Value: moID}) + + freq, err := vcenter.ClusterMinCPUFreq(ctx, ccr) + if err != nil { + errs = append(errs, err) + } else if minFreq == 0 || freq < minFreq { + minFreq = freq + } + } + } + + return minFreq, k8serrors.NewAggregate(errs) +} + +func (vs *vSphereVMProvider) GetTasksByActID(ctx goctx.Context, actID string) (_ []types.TaskInfo, retErr error) { + vcClient, err := vs.getVcClient(ctx) + if err != nil { + return nil, err + } + + taskManager := task.NewManager(vcClient.VimClient()) + filterSpec := types.TaskFilterSpec{ + ActivationId: []string{actID}, + } + + collector, err := taskManager.CreateCollectorForTasks(ctx, filterSpec) + if err != nil { + return nil, errors.Wrapf(err, "failed to create collector for tasks") + } + defer func() { + err = collector.Destroy(ctx) + if retErr == nil { + retErr = err + } + }() + + taskList := make([]types.TaskInfo, 0) + for { + nextTasks, err := collector.ReadNextTasks(ctx, taskHistoryCollectorPageSize) + if err != nil { + log.Error(err, "failed to read next tasks") + return nil, err + } + if len(nextTasks) == 0 { + break + } + taskList = append(taskList, nextTasks...) + } + + log.V(5).Info("found tasks", "actID", actID, "tasks", taskList) + return taskList, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go new file mode 100644 index 000000000..aae8db2f4 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy.go @@ -0,0 +1,261 @@ +// Copyright (c) 2020-2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + "context" + "fmt" + + vimtypes "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/clustermodules" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" +) + +// IsVirtualMachineSetResourcePolicyReady checks if the VirtualMachineSetResourcePolicy for the AZ is ready. +func (vs *vSphereVMProvider) IsVirtualMachineSetResourcePolicyReady( + ctx context.Context, + azName string, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (bool, error) { + + client, err := vs.getVcClient(ctx) + if err != nil { + return false, err + } + + folderMoID, rpMoID, err := topology.GetNamespaceFolderAndRPMoID(ctx, vs.k8sClient, azName, resourcePolicy.Namespace) + if err != nil { + return false, err + } + + folderExists, err := vcenter.DoesChildFolderExist(ctx, client.VimClient(), folderMoID, resourcePolicy.Spec.Folder) + if err != nil { + return false, err + } + + rpExists, err := vcenter.DoesChildResourcePoolExist(ctx, client.VimClient(), rpMoID, resourcePolicy.Spec.ResourcePool.Name) + if err != nil { + return false, err + } + + clusterRef, err := vcenter.GetResourcePoolOwnerMoRef(ctx, client.VimClient(), rpMoID) + if err != nil { + return false, err + } + + modulesExist, err := vs.doClusterModulesExist(ctx, client.ClusterModuleClient(), clusterRef.Reference(), resourcePolicy) + if err != nil { + return false, err + } + + if !rpExists || !folderExists || !modulesExist { + log.V(4).Info("Resource policy is not ready", "resourcePolicy", resourcePolicy.Name, + "namespace", resourcePolicy.Name, "az", azName, "resourcePool", rpExists, "folder", folderExists, "modules", modulesExist) + return false, nil + } + + return true, nil +} + +// CreateOrUpdateVirtualMachineSetResourcePolicy creates if a VirtualMachineSetResourcePolicy doesn't exist, updates otherwise. +func (vs *vSphereVMProvider) CreateOrUpdateVirtualMachineSetResourcePolicy( + ctx context.Context, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + folderMoID, rpMoIDs, err := vs.getNamespaceFolderAndRPMoIDs(ctx, resourcePolicy.Namespace) + if err != nil { + return err + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return err + } + + vimClient := client.VimClient() + var errs []error + + _, err = vcenter.CreateFolder(ctx, vimClient, folderMoID, resourcePolicy.Spec.Folder) + if err != nil { + errs = append(errs, err) + } + + for _, rpMoID := range rpMoIDs { + _, err := vcenter.CreateOrUpdateChildResourcePool(ctx, vimClient, rpMoID, &resourcePolicy.Spec.ResourcePool) + if err != nil { + errs = append(errs, err) + } + + clusterRef, err := vcenter.GetResourcePoolOwnerMoRef(ctx, vimClient, rpMoID) + if err == nil { + err = vs.createClusterModules(ctx, client.ClusterModuleClient(), clusterRef.Reference(), resourcePolicy) + } + if err != nil { + errs = append(errs, err) + } + } + + return k8serrors.NewAggregate(errs) +} + +// DeleteVirtualMachineSetResourcePolicy deletes the VirtualMachineSetPolicy. +func (vs *vSphereVMProvider) DeleteVirtualMachineSetResourcePolicy( + ctx context.Context, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + folderMoID, rpMoIDs, err := vs.getNamespaceFolderAndRPMoIDs(ctx, resourcePolicy.Namespace) + if err != nil { + return err + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return err + } + + vimClient := client.VimClient() + var errs []error + + for _, rpMoID := range rpMoIDs { + err := vcenter.DeleteChildResourcePool(ctx, vimClient, rpMoID, resourcePolicy.Spec.ResourcePool.Name) + if err != nil { + errs = append(errs, err) + } + } + + errs = append(errs, vs.deleteClusterModules(ctx, client.ClusterModuleClient(), resourcePolicy)...) + + if err := vcenter.DeleteChildFolder(ctx, vimClient, folderMoID, resourcePolicy.Spec.Folder); err != nil { + errs = append(errs, err) + } + + return k8serrors.NewAggregate(errs) +} + +// doClusterModulesExist checks whether all the ClusterModules for the given VirtualMachineSetResourcePolicy +// have been created and exist in VC for the Session's Cluster. +func (vs *vSphereVMProvider) doClusterModulesExist( + ctx context.Context, + clusterModProvider clustermodules.Provider, + clusterRef vimtypes.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) (bool, error) { + + for _, groupName := range resourcePolicy.Spec.ClusterModuleGroups { + _, moduleID := clustermodules.FindClusterModuleUUID(groupName, clusterRef, resourcePolicy) + if moduleID == "" { + return false, nil + } + + exists, err := clusterModProvider.DoesModuleExist(ctx, moduleID, clusterRef) + if !exists || err != nil { + return false, err + } + } + + return true, nil +} + +// createClusterModules creates all the ClusterModules that has not created yet for a +// given VirtualMachineSetResourcePolicy in VC. +func (vs *vSphereVMProvider) createClusterModules( + ctx context.Context, + clusterModProvider clustermodules.Provider, + clusterRef vimtypes.ManagedObjectReference, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) error { + + var errs []error + + // There is no way to give a name when creating a VC cluster module, so we have to + // resort to using the status as the source of truth. This can result in orphaned + // modules if, for instance, we fail to update the resource policy k8s object. + for _, groupName := range resourcePolicy.Spec.ClusterModuleGroups { + idx, moduleID := clustermodules.FindClusterModuleUUID(groupName, clusterRef, resourcePolicy) + + if moduleID != "" { + // Verify this cluster module exists on VC for this cluster. + exists, err := clusterModProvider.DoesModuleExist(ctx, moduleID, clusterRef) + if err != nil { + errs = append(errs, err) + continue + } + if !exists { + // Status entry is stale. Create below. + moduleID = "" + } + } else { + var err error + // See if there is already a module for this cluster but without the ClusterMoID field + // set that we can claim. + idx, moduleID, err = clustermodules.ClaimClusterModuleUUID(ctx, clusterModProvider, + groupName, clusterRef, resourcePolicy) + if err != nil { + errs = append(errs, err) + continue + } + } + + if moduleID == "" { + var err error + moduleID, err = clusterModProvider.CreateModule(ctx, clusterRef) + if err != nil { + errs = append(errs, err) + continue + } + } + + if idx >= 0 { + resourcePolicy.Status.ClusterModules[idx].ModuleUuid = moduleID + resourcePolicy.Status.ClusterModules[idx].ClusterMoID = clusterRef.Value + } else { + status := vmopv1.VSphereClusterModuleStatus{ + GroupName: groupName, + ModuleUuid: moduleID, + ClusterMoID: clusterRef.Value, + } + resourcePolicy.Status.ClusterModules = append(resourcePolicy.Status.ClusterModules, status) + } + } + + return k8serrors.NewAggregate(errs) +} + +// deleteClusterModules deletes all the ClusterModules associated with a given VirtualMachineSetResourcePolicy in VC. +func (vs *vSphereVMProvider) deleteClusterModules( + ctx context.Context, + clusterModProvider clustermodules.Provider, + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy) []error { + + var errModStatus []vmopv1.VSphereClusterModuleStatus + var errs []error + + for _, moduleStatus := range resourcePolicy.Status.ClusterModules { + err := clusterModProvider.DeleteModule(ctx, moduleStatus.ModuleUuid) + if err != nil { + errModStatus = append(errModStatus, moduleStatus) + errs = append(errs, err) + } + } + + resourcePolicy.Status.ClusterModules = errModStatus + return errs +} + +func (vs *vSphereVMProvider) getNamespaceFolderAndRPMoIDs( + ctx context.Context, + namespace string) (string, []string, error) { + + folderMoID, rpMoIDs, err := topology.GetNamespaceFolderAndRPMoIDs(ctx, vs.k8sClient, namespace) + if err != nil { + return "", nil, err + } + + if folderMoID == "" { + return "", nil, fmt.Errorf("namespace %s not present in any AvailabilityZones", namespace) + } + + return folderMoID, rpMoIDs, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go new file mode 100644 index 000000000..03c76f704 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_resourcepolicy_test.go @@ -0,0 +1,234 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "fmt" + "path" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/object" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func getVirtualMachineSetResourcePolicy(name, namespace string) *vmopv1.VirtualMachineSetResourcePolicy { + return &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-resourcepolicy", name), + Namespace: namespace, + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{ + Name: fmt.Sprintf("%s-resourcepool", name), + Reservations: vmopv1.VirtualMachineResourceSpec{}, + Limits: vmopv1.VirtualMachineResourceSpec{}, + }, + Folder: fmt.Sprintf("%s-folder", name), + ClusterModuleGroups: []string{"ControlPlane", "NodeGroup1"}, + }, + } +} + +func resourcePolicyTests() { + Describe("VirtualMachineSetResourcePolicy Tests", func() { + + var ( + initObjects []client.Object + ctx *builder.TestContextForVCSim + nsInfo builder.WorkloadNamespaceInfo + testConfig builder.VCSimTestConfig + vmProvider vmprovider.VirtualMachineProviderInterfaceA2 + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + + nsInfo = ctx.CreateWorkloadNamespace() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + }) + + Context("VirtualMachineSetResourcePolicy", func() { + var ( + resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ) + + JustBeforeEach(func() { + testPolicyName := "test-policy" + + resourcePolicy = getVirtualMachineSetResourcePolicy(testPolicyName, nsInfo.Namespace) + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + }) + + JustAfterEach(func() { + Expect(vmProvider.DeleteVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(resourcePolicy.Status.ClusterModules).Should(BeEmpty()) + }) + + It("creates expected cluster modules", func() { + modules := resourcePolicy.Status.ClusterModules + Expect(modules).Should(HaveLen(2)) + module := modules[0] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[0])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + module = modules[1] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[1])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + }) + + Context("for an existing resource policy", func() { + It("should keep existing cluster modules", func() { + status := resourcePolicy.Status.DeepCopy() + + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(resourcePolicy.Status.ClusterModules).To(ContainElements(status.ClusterModules)) + }) + + It("successfully able to find the resource policy", func() { + exists, err := vmProvider.IsVirtualMachineSetResourcePolicyReady(ctx, "", resourcePolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + }) + + Context("for an absent resource policy", func() { + It("should fail to find the resource policy without any errors", func() { + failResPolicy := getVirtualMachineSetResourcePolicy("test-policy", nsInfo.Namespace) + exists, err := vmProvider.IsVirtualMachineSetResourcePolicyReady(ctx, "", failResPolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeFalse()) + }) + }) + + Context("for a resource policy with invalid cluster module", func() { + It("successfully able to delete the resource policy", func() { + resourcePolicy.Status.ClusterModules = append([]vmopv1.VSphereClusterModuleStatus{ + { + GroupName: "invalid-group", + ModuleUuid: "invalid-uuid", + }, + }, resourcePolicy.Status.ClusterModules...) + }) + }) + + It("creates expected resource pool", func() { + rp, err := ctx.GetSingleClusterCompute().ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + + // Make trip through the Finder to populate InventoryPath. + objRef, err := ctx.Finder.ObjectReference(ctx, rp.Reference()) + Expect(err).ToNot(HaveOccurred()) + rp, ok := objRef.(*object.ResourcePool) + Expect(ok).To(BeTrue()) + + inventoryPath := path.Join(rp.InventoryPath, nsInfo.Namespace, resourcePolicy.Spec.ResourcePool.Name) + _, err = ctx.Finder.ResourcePool(ctx, inventoryPath) + Expect(err).ToNot(HaveOccurred()) + }) + + It("creates expected child folder", func() { + _, err := ctx.Finder.Folder(ctx, path.Join(nsInfo.Folder.InventoryPath, resourcePolicy.Spec.Folder)) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("when HA is enabled", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + }) + + It("creates expected cluster modules for each cluster", func() { + moduleCount := len(resourcePolicy.Spec.ClusterModuleGroups) + Expect(moduleCount).To(Equal(2)) + + modules := resourcePolicy.Status.ClusterModules + zoneNames := ctx.ZoneNames + Expect(modules).To(HaveLen(len(zoneNames) * ctx.ClustersPerZone * moduleCount)) + + for zoneIdx, zoneName := range zoneNames { + // NOTE: This assumes some ordering but is the easiest way to test. + moduleIdx := zoneIdx * ctx.ClustersPerZone * moduleCount + modules := modules[moduleIdx : moduleIdx+ctx.ClustersPerZone*moduleCount] + + ccrs := ctx.GetAZClusterComputes(zoneName) + Expect(ccrs).To(HaveLen(ctx.ClustersPerZone)) + + for _, cluster := range ccrs { + clusterMoID := cluster.Reference().Value + + module := modules[0] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[0])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + Expect(module.ClusterMoID).To(Equal(clusterMoID)) + + module = modules[1] + Expect(module.GroupName).To(Equal(resourcePolicy.Spec.ClusterModuleGroups[1])) + Expect(module.ModuleUuid).ToNot(BeEmpty()) + Expect(module.ClusterMoID).To(Equal(clusterMoID)) + } + } + }) + + It("should claim cluster module without ClusterMoID set", func() { + Expect(resourcePolicy.Spec.ClusterModuleGroups).ToNot(BeEmpty()) + groupName := resourcePolicy.Spec.ClusterModuleGroups[0] + + moduleStatus := resourcePolicy.Status.DeepCopy() + Expect(moduleStatus.ClusterModules).ToNot(BeEmpty()) + + for i := range resourcePolicy.Status.ClusterModules { + if resourcePolicy.Status.ClusterModules[i].GroupName == groupName { + resourcePolicy.Status.ClusterModules[i].ClusterMoID = "" + } + } + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(resourcePolicy.Status.ClusterModules).To(Equal(moduleStatus.ClusterModules)) + }) + + It("successfully able to find the resource policy in each zone", func() { + for _, zoneName := range ctx.ZoneNames { + exists, err := vmProvider.IsVirtualMachineSetResourcePolicyReady(ctx, zoneName, resourcePolicy) + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + } + }) + + It("creates expected resource pool for each cluster", func() { + for _, zoneName := range ctx.ZoneNames { + for _, cluster := range ctx.GetAZClusterComputes(zoneName) { + rp, err := cluster.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + + // Make trip through the Finder to populate InventoryPath. + objRef, err := ctx.Finder.ObjectReference(ctx, rp.Reference()) + Expect(err).ToNot(HaveOccurred()) + rp, ok := objRef.(*object.ResourcePool) + Expect(ok).To(BeTrue()) + + inventoryPath := path.Join(rp.InventoryPath, nsInfo.Namespace, resourcePolicy.Spec.ResourcePool.Name) + _, err = ctx.Finder.ResourcePool(ctx, inventoryPath) + Expect(err).ToNot(HaveOccurred()) + } + } + }) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_test.go new file mode 100644 index 000000000..5a447c69d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_test.go @@ -0,0 +1,88 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "sync" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func cpuFreqTests() { + + var ( + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + vmProvider vmprovider.VirtualMachineProviderInterface + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + vmProvider = nil + }) + + Context("ComputeCPUMinFrequency", func() { + It("returns success", func() { + Expect(vmProvider.ComputeCPUMinFrequency(ctx)).To(Succeed()) + }) + }) +} + +func initOvfCacheAndLockPoolTests() { + + var ( + expireAfter = 3 * time.Second + checkExpireInterval = 1 * time.Second + maxItems = 3 + + ovfCache *util.Cache[vsphere.VersionedOVFEnvelope] + ovfLockPool *util.LockPool[string, *sync.RWMutex] + ) + + BeforeEach(func() { + ovfCache, ovfLockPool = vsphere.InitOvfCacheAndLockPool( + expireAfter, checkExpireInterval, maxItems) + }) + + AfterEach(func() { + ovfCache = nil + ovfLockPool = nil + }) + + Context("InitOvfCacheAndLockPool", func() { + It("should clean up lock pool when the item is expired in cache", func() { + Expect(ovfCache).ToNot(BeNil()) + Expect(ovfLockPool).ToNot(BeNil()) + + itemID := "test-item-id" + res := ovfCache.Put(itemID, vsphere.VersionedOVFEnvelope{}) + Expect(res).To(Equal(util.CachePutResultCreate)) + curItemLock := ovfLockPool.Get(itemID) + Expect(curItemLock).ToNot(BeNil()) + + Eventually(func() bool { + // ovfLockPool.Get() returns a new lock if the item key is not found. + // So the lock should be different when the item is expired and deleted from pool. + return ovfLockPool.Get(itemID) != curItemLock + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go new file mode 100644 index 000000000..fb7721812 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go @@ -0,0 +1,1169 @@ +// Copyright (c) 2022 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + goctx "context" + "fmt" + "strings" + "sync" + "text/template" + + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" + + "github.com/pkg/errors" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + k8serrors "k8s.io/apimachinery/pkg/util/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/util" + vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/client" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/contentlibrary" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/placement" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/storage" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vcenter" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" +) + +// VMCreateArgs contains the arguments needed to create a VM on VC. +type VMCreateArgs struct { + vmlifecycle.CreateArgs + vmlifecycle.BootstrapData + + VMClass *vmopv1.VirtualMachineClass + ResourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ImageObj ctrlclient.Object + ImageSpec *vmopv1.VirtualMachineImageSpec + ImageStatus *vmopv1.VirtualMachineImageStatus + + StorageClassesToIDs map[string]string + HasInstanceStorage bool + ChildResourcePoolName string + ChildFolderName string + ClusterMoRef types.ManagedObjectReference + + NetworkResults network.NetworkInterfaceResults +} + +// TODO: Until we sort out what the Session becomes. +type vmUpdateArgs = session.VMUpdateArgs + +const ( + FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" +) + +var ( + createCountLock sync.Mutex + concurrentCreateCount int + + // SkipVMImageCLProviderCheck skips the checks that a VM Image has a Content Library item provider + // since a VirtualMachineImage created for a VM template won't have either. This has been broken for + // a long time but was otherwise masked on how the tests used to be organized. + SkipVMImageCLProviderCheck = false +) + +func (vs *vSphereVMProvider) CreateOrUpdateVirtualMachine( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) error { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "createOrUpdateVM")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return err + } + + vcVM, err := vs.getVM(vmCtx, client, false) + if err != nil { + return err + } + + if vcVM == nil { + var createArgs *VMCreateArgs + + vcVM, createArgs, err = vs.createVirtualMachine(vmCtx, client) + if err != nil { + return err + } + + if vcVM == nil { + // Creation was not ready or blocked for some reason. We depend on the controller + // to eventually retry the create. + return nil + } + + return vs.createdVirtualMachineFallthroughUpdate(vmCtx, vcVM, client, createArgs) + } + + return vs.updateVirtualMachine(vmCtx, vcVM, client, nil) +} + +func (vs *vSphereVMProvider) DeleteVirtualMachine( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) error { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "deleteVM")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return err + } + + vcVM, err := vs.getVM(vmCtx, client, false) + if err != nil { + return err + } else if vcVM == nil { + // VM does not exist. + return nil + } + + return virtualmachine.DeleteVirtualMachine(vmCtx, vcVM) +} + +func (vs *vSphereVMProvider) PublishVirtualMachine( + ctx goctx.Context, + vm *vmopv1.VirtualMachine, + vmPub *vmopv1.VirtualMachinePublishRequest, + cl *imgregv1a1.ContentLibrary, + actID string) (string, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: ctx, + // Update logger info + Logger: log.WithValues("vmName", vm.NamespacedName()). + WithValues("clName", fmt.Sprintf("%s/%s", cl.Namespace, cl.Name)). + WithValues("vmPubName", fmt.Sprintf("%s/%s", vmPub.Namespace, vmPub.Name)), + VM: vm, + } + + client, err := vs.getVcClient(ctx) + if err != nil { + return "", errors.Wrapf(err, "failed to get vCenter client") + } + + itemID, err := virtualmachine.CreateOVF(vmCtx, client.RestClient(), vmPub, cl, actID) + if err != nil { + return "", err + } + + return itemID, nil +} + +// BackupVirtualMachine backs up the VM data required for restore. +func (vs *vSphereVMProvider) BackupVirtualMachine(ctx goctx.Context, vm *vmopv1.VirtualMachine) error { + // TODO + return nil +} + +func (vs *vSphereVMProvider) GetVirtualMachineGuestHeartbeat( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "heartbeat")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return "", err + } + + vcVM, err := vs.getVM(vmCtx, client, true) + if err != nil { + return "", err + } + + status, err := virtualmachine.GetGuestHeartBeatStatus(vmCtx, vcVM) + if err != nil { + return "", err + } + + return status, nil +} + +func (vs *vSphereVMProvider) GetVirtualMachineWebMKSTicket( + ctx goctx.Context, + vm *vmopv1.VirtualMachine, + pubKey string) (string, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "webconsole")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return "", err + } + + vcVM, err := vs.getVM(vmCtx, client, true) + if err != nil { + return "", err + } + + ticket, err := virtualmachine.GetWebConsoleTicket(vmCtx, vcVM, pubKey) + if err != nil { + return "", err + } + + return ticket, nil +} + +func (vs *vSphereVMProvider) GetVirtualMachineHardwareVersion( + ctx goctx.Context, + vm *vmopv1.VirtualMachine) (int32, error) { + + vmCtx := context.VirtualMachineContextA2{ + Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "hardware-version")), + Logger: log.WithValues("vmName", vm.NamespacedName()), + VM: vm, + } + + client, err := vs.getVcClient(vmCtx) + if err != nil { + return 0, err + } + + vcVM, err := vs.getVM(vmCtx, client, true) + if err != nil { + return 0, err + } + + var o mo.VirtualMachine + err = vcVM.Properties(vmCtx, vcVM.Reference(), []string{"config.version"}, &o) + if err != nil { + return 0, err + } + + return contentlibrary.ParseVirtualHardwareVersion(o.Config.Version), nil +} + +func (vs *vSphereVMProvider) createVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client) (*object.VirtualMachine, *VMCreateArgs, error) { + + createArgs, err := vs.vmCreateGetArgs(vmCtx, vcClient) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateDoPlacement(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateGetFolderAndRPMoIDs(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateFixupConfigSpec(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + err = vs.vmCreateIsReady(vmCtx, vcClient, createArgs) + if err != nil { + return nil, nil, err + } + + // BMV: This is about where we used to do this check but it prb make more sense to do + // earlier, as to limit wasted work. Before DoPlacement() is likely the best place so + // the window between the placement decision and creating the VM on VC is small(ish). + allowed, createDeferFn, err := vs.vmCreateConcurrentAllowed(vmCtx) + if err != nil { + return nil, nil, err + } else if !allowed { + return nil, nil, nil + } + defer createDeferFn() + + moRef, err := vmlifecycle.CreateVirtualMachine( + vmCtx, + vcClient.ContentLibClient(), + vcClient.RestClient(), + vcClient.Finder(), + &createArgs.CreateArgs) + if err != nil { + vmCtx.Logger.Error(err, "CreateVirtualMachine failed") + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionCreated, "Error", err.Error()) + return nil, nil, err + } + + vmCtx.VM.Status.UniqueID = moRef.Reference().Value + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionCreated) + + return object.NewVirtualMachine(vcClient.VimClient(), *moRef), createArgs, nil +} + +func (vs *vSphereVMProvider) createdVirtualMachineFallthroughUpdate( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + // TODO: In the common case, we'll call directly into update right after create succeeds, and + // can use the createArgs to avoid doing a bunch of lookup work again. + + return vs.updateVirtualMachine(vmCtx, vcVM, vcClient, createArgs) +} + +func (vs *vSphereVMProvider) updateVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + vmCtx.Logger.V(4).Info("Updating VirtualMachine") + + { + // Hack - create just enough of the Session that's needed for update + + cluster, err := virtualmachine.GetVMClusterComputeResource(vmCtx, vcVM) + if err != nil { + return err + } + + ses := &session.Session{ + K8sClient: vs.k8sClient, + Client: vcClient, + Finder: vcClient.Finder(), + Cluster: cluster, + } + + getUpdateArgsFn := func() (*vmUpdateArgs, error) { + // TODO: Use createArgs if we already got them + _ = createArgs + return vs.vmUpdateGetArgs(vmCtx) + } + + err = ses.UpdateVirtualMachine(vmCtx, vcVM, getUpdateArgsFn) + if err != nil { + return err + } + } + + return nil +} + +// vmCreateDoPlacement determines placement of the VM prior to creating the VM on VC. +func (vs *vSphereVMProvider) vmCreateDoPlacement( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + placementConfigSpec := virtualmachine.CreateConfigSpecForPlacement( + vmCtx, + createArgs.ConfigSpec, + createArgs.StorageClassesToIDs) + + result, err := placement.Placement( + vmCtx, + vs.k8sClient, + vcClient.VimClient(), + placementConfigSpec, + createArgs.ChildResourcePoolName) + if err != nil { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionPlacementReady, "NotReady", err.Error()) + return err + } + + if result.PoolMoRef.Value != "" { + createArgs.ResourcePoolMoID = result.PoolMoRef.Value + } + + if result.HostMoRef != nil { + createArgs.HostMoID = result.HostMoRef.Value + } + + if result.InstanceStoragePlacement { + hostMoID := createArgs.HostMoID + + if hostMoID == "" { + return fmt.Errorf("placement result missing host required for instance storage") + } + + hostFQDN, err := vcenter.GetESXHostFQDN(vmCtx, vcClient.VimClient(), hostMoID) + if err != nil { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionPlacementReady, "NotReady", err.Error()) + return err + } + + if vmCtx.VM.Annotations == nil { + vmCtx.VM.Annotations = map[string]string{} + } + vmCtx.VM.Annotations[constants.InstanceStorageSelectedNodeMOIDAnnotationKey] = hostMoID + vmCtx.VM.Annotations[constants.InstanceStorageSelectedNodeAnnotationKey] = hostFQDN + } + + if result.ZonePlacement { + if vmCtx.VM.Labels == nil { + vmCtx.VM.Labels = map[string]string{} + } + // Note if the VM create fails for some reason, but this label gets updated on the k8s VM, + // then this is the pre-assigned zone on later create attempts. + vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey] = result.ZoneName + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionPlacementReady) + + return nil +} + +// vmCreateGetFolderAndRPMoIDs gets the MoIDs of the Folder and Resource Pool the VM will be created under. +func (vs *vSphereVMProvider) vmCreateGetFolderAndRPMoIDs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + if createArgs.ResourcePoolMoID == "" { + // We did not do placement so find this namespace/zone ResourcePool and Folder. + + nsFolderMoID, rpMoID, err := topology.GetNamespaceFolderAndRPMoID(vmCtx, vs.k8sClient, + vmCtx.VM.Labels[topology.KubernetesTopologyZoneLabelKey], vmCtx.VM.Namespace) + if err != nil { + return err + } + + // If this VM has a ResourcePolicy ResourcePool, lookup the child ResourcePool under the + // namespace/zone's root ResourcePool. This will be the VM's ResourcePool. + if createArgs.ChildResourcePoolName != "" { + parentRP := object.NewResourcePool(vcClient.VimClient(), + types.ManagedObjectReference{Type: "ResourcePool", Value: rpMoID}) + + childRP, err := vcenter.GetChildResourcePool(vmCtx, parentRP, createArgs.ChildResourcePoolName) + if err != nil { + return err + } + + rpMoID = childRP.Reference().Value + } + + createArgs.ResourcePoolMoID = rpMoID + createArgs.FolderMoID = nsFolderMoID + + } else { + // Placement already selected the ResourcePool/Cluster, so we just need this namespace's Folder. + nsFolderMoID, err := topology.GetNamespaceFolderMoID(vmCtx, vs.k8sClient, vmCtx.VM.Namespace) + if err != nil { + return err + } + + createArgs.FolderMoID = nsFolderMoID + } + + // If this VM has a ResourcePolicy Folder, lookup the child Folder under the namespace's Folder. + // This will be the VM's parent Folder in the VC inventory. + if createArgs.ChildFolderName != "" { + parentFolder := object.NewFolder(vcClient.VimClient(), + types.ManagedObjectReference{Type: "Folder", Value: createArgs.FolderMoID}) + + childFolder, err := vcenter.GetChildFolder(vmCtx, parentFolder, createArgs.ChildFolderName) + if err != nil { + return err + } + + createArgs.FolderMoID = childFolder.Reference().Value + } + + // Now that we know the ResourcePool, use that to look up the CCR. + clusterMoRef, err := vcenter.GetResourcePoolOwnerMoRef(vmCtx, vcClient.VimClient(), createArgs.ResourcePoolMoID) + if err != nil { + return err + } + createArgs.ClusterMoRef = clusterMoRef + + return nil +} + +func (vs *vSphereVMProvider) vmCreateFixupConfigSpec( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + fixedUp, err := network.ResolveBackingPostPlacement( + vmCtx, + vcClient.VimClient(), + createArgs.ClusterMoRef, + &createArgs.NetworkResults) + if err != nil { + return err + } + + if fixedUp { + // Now that the backing is resolved for this CCR, re-zip to update the ConfigSpec. What a mess. + err = vs.vmCreateGenConfigSpecZipNetworkInterfaces(vmCtx, createArgs) + if err != nil { + return err + } + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateIsReady( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + if policy := createArgs.ResourcePolicy; policy != nil { + // TODO: May want to do this as to filter the placement candidates. + exists, err := vs.doClusterModulesExist(vmCtx, vcClient.ClusterModuleClient(), createArgs.ClusterMoRef, policy) + if err != nil { + return err + } else if !exists { + return fmt.Errorf("VirtualMachineSetResourcePolicy cluster module is not ready") + } + } + + if createArgs.HasInstanceStorage { + if _, ok := vmCtx.VM.Annotations[constants.InstanceStoragePVCsBoundAnnotationKey]; !ok { + return fmt.Errorf("instance storage PVCs are not bound yet") + } + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateConcurrentAllowed(vmCtx context.VirtualMachineContextA2) (bool, func(), error) { + maxDeployThreads, ok := vmCtx.Value(context.MaxDeployThreadsContextKey).(int) + if !ok { + return false, nil, fmt.Errorf("MaxDeployThreadsContextKey missing from context") + } + + createCountLock.Lock() + if concurrentCreateCount >= maxDeployThreads { + createCountLock.Unlock() + vmCtx.Logger.Info("Too many create VirtualMachine already occurring. Re-queueing request") + return false, nil, nil + } + + concurrentCreateCount++ + createCountLock.Unlock() + + decrementFn := func() { + createCountLock.Lock() + concurrentCreateCount-- + createCountLock.Unlock() + } + + return true, decrementFn, nil +} + +func (vs *vSphereVMProvider) vmCreateGetArgs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client) (*VMCreateArgs, error) { + + createArgs, err := vs.vmCreateGetPrereqs(vmCtx, vcClient) + if err != nil { + return nil, err + } + + err = vs.vmCreateDoNetworking(vmCtx, vcClient, createArgs) + if err != nil { + return nil, err + } + + err = vs.vmCreateGenConfigSpec(vmCtx, createArgs) + if err != nil { + return nil, err + } + + err = vs.vmCreateValidateArgs(vmCtx, vcClient, createArgs) + if err != nil { + return nil, err + } + + return createArgs, nil +} + +// vmCreateGetPrereqs returns the VMCreateArgs populated with the k8s objects required to +// create the VM on VC. +func (vs *vSphereVMProvider) vmCreateGetPrereqs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client) (*VMCreateArgs, error) { + + createArgs := &VMCreateArgs{} + var prereqErrs []error + + if err := vs.vmCreateGetVirtualMachineClass(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetVirtualMachineImage(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetSetResourcePolicy(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetBootstrap(vmCtx, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + if err := vs.vmCreateGetStoragePrereqs(vmCtx, vcClient, createArgs); err != nil { + prereqErrs = append(prereqErrs, err) + } + + // This is about the point where historically we'd declare the prereqs ready or not. There + // is still a lot of work to do - and things to fail - before the actual create, but there + // is no point in continuing if the above checks aren't met since we are missing data + // required to create the VM. + if len(prereqErrs) > 0 { + return nil, k8serrors.NewAggregate(prereqErrs) + } + + // Note that once the VM is created, it is hard for us to later resolve what image was used, + // since a NS or cluster scoped image could have been created or deleted. + vmCtx.VM.Status.Image = &common.LocalObjectRef{ + APIVersion: createArgs.ImageObj.GetObjectKind().GroupVersionKind().Version, + Kind: createArgs.ImageObj.GetObjectKind().GroupVersionKind().Kind, + Name: createArgs.ImageObj.GetName(), + } + + vmCtx.VM.Status.Class = &common.LocalObjectRef{ + APIVersion: createArgs.VMClass.APIVersion, + Kind: createArgs.VMClass.Kind, + Name: createArgs.VMClass.Name, + } + + return createArgs, nil +} + +func (vs *vSphereVMProvider) vmCreateGetVirtualMachineClass( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + vmClass, err := GetVirtualMachineClass(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + createArgs.VMClass = vmClass + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetVirtualMachineImage( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + imageObj, imageSpec, imageStatus, err := GetVirtualMachineImageSpecAndStatus(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + createArgs.ImageObj = imageObj + createArgs.ImageSpec = imageSpec + createArgs.ImageStatus = imageStatus + + // This is clunky, but we need to know how to use the image to create the VM. Our only supported + // method is via the ContentLibrary, so check if this image was derived from a CL item. + switch imageSpec.ProviderRef.Kind { + case "ClusterContentLibraryItem", "ContentLibraryItem": + createArgs.UseContentLibrary = true + createArgs.ProviderItemID = imageStatus.ProviderItemID + default: + if !SkipVMImageCLProviderCheck { + err := fmt.Errorf("unsupported image provider kind: %s", imageSpec.ProviderRef.Kind) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady, "NotSupported", err.Error()) + return err + } + // Testing only: we'll clone the source VM found in the Inventory. + createArgs.UseContentLibrary = false + createArgs.ProviderItemID = vmCtx.VM.Spec.ImageName + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetSetResourcePolicy( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + resourcePolicy, err := GetVMSetResourcePolicy(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + // The SetResourcePolicy is optional (TKG VMs will always have it). + if resourcePolicy != nil { + createArgs.ResourcePolicy = resourcePolicy + createArgs.ChildFolderName = resourcePolicy.Spec.Folder + createArgs.ChildResourcePoolName = resourcePolicy.Spec.ResourcePool.Name + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetBootstrap( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + data, vAppData, vAppExData, err := GetVirtualMachineBootstrap(vmCtx, vs.k8sClient) + if err != nil { + return err + } + + createArgs.BootstrapData.Data = data + createArgs.BootstrapData.VAppData = vAppData + createArgs.BootstrapData.VAppExData = vAppExData + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGetStoragePrereqs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + if lib.IsInstanceStorageFSSEnabled() { + // To determine all the storage profiles, we need the class because of the possibility of + // InstanceStorage volumes. If we weren't able to get the class earlier, still check & set + // the storage condition because instance storage usage is rare, it is helpful to report + // as many prereqs as possible, and we'll reevaluate this once the class is available. + if createArgs.VMClass != nil { + // Add the class's instance storage disks - if any - to the VM.Spec. Once the instance + // storage disks are added to the VM, they are set in stone even if the class itself or + // the VM's assigned class changes. + createArgs.HasInstanceStorage = AddInstanceStorageVolumes(vmCtx, createArgs.VMClass) + } + } + + storageClassesToIDs, err := storage.GetVMStoragePoliciesIDs(vmCtx, vs.k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionStorageReady, reason, msg) + return err + } + + vmStorageProfileID := storageClassesToIDs[vmCtx.VM.Spec.StorageClass] + + provisioningType, err := virtualmachine.GetDefaultDiskProvisioningType(vmCtx, vcClient, vmStorageProfileID) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionStorageReady, reason, msg) + return err + } + + createArgs.StorageClassesToIDs = storageClassesToIDs + createArgs.StorageProvisioning = provisioningType + createArgs.StorageProfileID = vmStorageProfileID + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionStorageReady) + + return nil +} + +func (vs *vSphereVMProvider) vmCreateDoNetworking( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + networkSpec := &vmCtx.VM.Spec.Network + if networkSpec.Disabled { + // No connected networking for this VM. Any EthCards will be removed later. + return nil + } + + interfaces := networkSpec.Interfaces + if len(interfaces) == 0 { + // VM gets one automatic NIC. Create the default interface from fields in the network spec. + defaultInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ + Name: networkSpec.DeviceName, + Addresses: networkSpec.Addresses, + DHCP4: networkSpec.DHCP4, + DHCP6: networkSpec.DHCP6, + Gateway4: networkSpec.Gateway4, + Gateway6: networkSpec.Gateway6, + MTU: networkSpec.MTU, + Nameservers: networkSpec.Nameservers, + Routes: networkSpec.Routes, + SearchDomains: networkSpec.SearchDomains, + } + + if defaultInterface.Name == "" { + defaultInterface.Name = "eth0" + } + if networkSpec.Network != nil { + defaultInterface.Network = *networkSpec.Network + } + + interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{defaultInterface} + } + + results, err := network.CreateAndWaitForNetworkInterfaces( + vmCtx, + vs.k8sClient, + vcClient.VimClient(), + vcClient.Finder(), + nil, // Don't know the CCR yet (needed to resolve backings for NSX-T) + interfaces) + if err != nil { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionNetworkReady, "NotReady", err.Error()) + return err + } + + createArgs.NetworkResults = results + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionNetworkReady) + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpec( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + // TODO: This is a partial dupe of what's done in the update path in the remaining Session code. I got + // tired of trying to keep that in sync so we get to live with a frankenstein thing longer. + + var vmClassConfigSpec *types.VirtualMachineConfigSpec + if rawConfigSpec := createArgs.VMClass.Spec.ConfigSpec; lib.IsVMClassAsConfigFSSDaynDateEnabled() && len(rawConfigSpec) > 0 { + configSpec, err := GetVMClassConfigSpec(rawConfigSpec) + if err != nil { + return err + } + vmClassConfigSpec = configSpec + } else { + vmClassConfigSpec = virtualmachine.ConfigSpecFromVMClassDevices(&createArgs.VMClass.Spec) + } + + var minCPUFreq uint64 + if res := createArgs.VMClass.Spec.Policies.Resources; !res.Requests.Cpu.IsZero() || !res.Limits.Cpu.IsZero() { + freq, err := vs.getOrComputeCPUMinFrequency(vmCtx) + if err != nil { + return err + } + minCPUFreq = freq + } + + createArgs.ConfigSpec = virtualmachine.CreateConfigSpec( + vmCtx, + vmClassConfigSpec, + &createArgs.VMClass.Spec, + createArgs.ImageStatus, + minCPUFreq) + + // TODO: This should be in CreateConfigSpec() + if createArgs.ConfigSpec.Version == "" { + imageVer := int32(0) + if createArgs.ImageStatus.HardwareVersion != nil { + imageVer = *createArgs.ImageStatus.HardwareVersion + } + + version := HardwareVersionForPVCandPCIDevices(imageVer, createArgs.ConfigSpec, HasPVC(vmCtx.VM.Spec)) + if version != 0 { + createArgs.ConfigSpec.Version = fmt.Sprintf("vmx-%d", version) + } + } + + err := vs.vmCreateGenConfigSpecExtraConfig(vmCtx, createArgs) + if err != nil { + return err + } + + err = vs.vmCreateGenConfigSpecChangeBootDiskSize(vmCtx, createArgs) + if err != nil { + return err + } + + err = vs.vmCreateGenConfigSpecZipNetworkInterfaces(vmCtx, createArgs) + if err != nil { + return err + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpecExtraConfig( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + ecMap := make(map[string]string, len(vs.globalExtraConfig)) + + // The only use of this template is for the JSON_EXTRA_CONFIG that is set in gce2e env + // to populate {{.ImageName }} so vcsim will create a container for the VM. + // BMV: This should be removable now that vcsim gce2e is gone. + renderTemplateFn := func(name, text string) string { + t, err := template.New(name).Parse(text) + if err != nil { + return text + } + b := strings.Builder{} + if err := t.Execute(&b, createArgs.ImageStatus); err != nil { + return text + } + return b.String() + } + for k, v := range vs.globalExtraConfig { + ecMap[k] = renderTemplateFn(k, v) + } + + hasPassthroughDevices := len(util.SelectVirtualPCIPassthrough(util.DevicesFromConfigSpec(createArgs.ConfigSpec))) > 0 + + if hasPassthroughDevices || createArgs.HasInstanceStorage { + ecMap[constants.MMPowerOffVMExtraConfigKey] = constants.ExtraConfigTrue + } + + if hasPassthroughDevices { + mmioSize := vmCtx.VM.Annotations[constants.PCIPassthruMMIOOverrideAnnotation] + if mmioSize == "" { + mmioSize = constants.PCIPassthruMMIOSizeDefault + } + if mmioSize != "0" { + ecMap[constants.PCIPassthruMMIOExtraConfigKey] = constants.ExtraConfigTrue + ecMap[constants.PCIPassthruMMIOSizeExtraConfigKey] = mmioSize + } + } + + // The ConfigSpec's current ExtraConfig values (that came from the class) take precedence over what was set here. + createArgs.ConfigSpec.ExtraConfig = util.AppendNewExtraConfigValues(createArgs.ConfigSpec.ExtraConfig, ecMap) + + // Leave constants.VMOperatorV1Alpha1ExtraConfigKey for the update path (if that's still even needed) + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpecChangeBootDiskSize( + vmCtx context.VirtualMachineContextA2, + _ *VMCreateArgs) error { + + capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity + if capacity.IsZero() { + return nil + } + + // TODO: How to we determine the DeviceKey for the DeviceChange entry? We probably have to + // crack the image/source, which is hard to do ATM. Punt on this for a placement consideration + // and we'll resize the boot (first) disk after VM create like before. + + return nil +} + +func (vs *vSphereVMProvider) vmCreateGenConfigSpecZipNetworkInterfaces( + vmCtx context.VirtualMachineContextA2, + createArgs *VMCreateArgs) error { + + if vmCtx.VM.Spec.Network.Disabled { + util.RemoveDevicesFromConfigSpec(createArgs.ConfigSpec, util.IsEthernetCard) + return nil + } + + resultsIdx := 0 + var unmatchedEthDevices []int + + for idx := range createArgs.ConfigSpec.DeviceChange { + spec := createArgs.ConfigSpec.DeviceChange[idx].GetVirtualDeviceConfigSpec() + if spec == nil || !util.IsEthernetCard(spec.Device) { + continue + } + + device := spec.Device + ethCard := device.(types.BaseVirtualEthernetCard).GetVirtualEthernetCard() + + if resultsIdx < len(createArgs.NetworkResults.Results) { + err := network.ApplyInterfaceResultToVirtualEthCard(vmCtx, ethCard, &createArgs.NetworkResults.Results[resultsIdx]) + if err != nil { + return err + } + resultsIdx++ + + } else { + // This ConfigSpec Ethernet device does not have a corresponding entry in the VM Spec, so we + // won't ever have a backing for it. Remove it from the ConfigSpec since that is the easiest + // thing to do, since extra NICs can cause later complications around GOSC and other customizations. + // The downside with this is that if a NIC is added to the VM Spec, it won't necessarily have this + // config but the default. Revisit this later if we don't like that behavior. + unmatchedEthDevices = append(unmatchedEthDevices, idx-len(unmatchedEthDevices)) + } + } + + if len(unmatchedEthDevices) > 0 { + deviceChange := createArgs.ConfigSpec.DeviceChange + for _, idx := range unmatchedEthDevices { + deviceChange = append(deviceChange[:idx], deviceChange[idx+1:]...) + } + createArgs.ConfigSpec.DeviceChange = deviceChange + } + + // Any remaining VM Spec network interfaces were not matched with a device in the ConfigSpec, so + // create a default virtual ethernet card for them. + for i := resultsIdx; i < len(createArgs.NetworkResults.Results); i++ { + ethCardDev, err := network.CreateDefaultEthCard(vmCtx, &createArgs.NetworkResults.Results[i]) + if err != nil { + return err + } + + // May not have the backing yet (NSX-T). We come back through here after placement once we + // know the backing. + if ethCardDev != nil { + createArgs.ConfigSpec.DeviceChange = append(createArgs.ConfigSpec.DeviceChange, &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: ethCardDev, + }) + } + } + + return nil +} + +func (vs *vSphereVMProvider) vmCreateValidateArgs( + vmCtx context.VirtualMachineContextA2, + vcClient *vcclient.Client, + createArgs *VMCreateArgs) error { + + // Some of this would be better done in the validation webhook but have it here for now. + cfg := vcClient.Config() + + if cfg.StorageClassRequired { + // In WCP this is always required. + if vmCtx.VM.Spec.StorageClass == "" { + return fmt.Errorf("StorageClass is required but not specified") + } + + if createArgs.StorageProfileID == "" { + // GetVMStoragePoliciesIDs() would have returned an error if the policy didn't exist, but + // ensure the field is set. + return fmt.Errorf("no StorageProfile found for StorageClass %s", vmCtx.VM.Spec.StorageClass) + } + + } else if vmCtx.VM.Spec.StorageClass == "" { + // This is only set in gce2e. + if cfg.Datastore == "" { + return fmt.Errorf("no Datastore provided in configuration") + } + + datastore, err := vcClient.Finder().Datastore(vmCtx, cfg.Datastore) + if err != nil { + return fmt.Errorf("failed to find Datastore %s: %w", cfg.Datastore, err) + } + + createArgs.DatastoreMoID = datastore.Reference().Value + } + + return nil +} + +func (vs *vSphereVMProvider) vmUpdateGetArgs( + vmCtx context.VirtualMachineContextA2) (*vmUpdateArgs, error) { + + vmClass, err := GetVirtualMachineClass(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + resourcePolicy, err := GetVMSetResourcePolicy(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + data, vAppData, vAppExData, err := GetVirtualMachineBootstrap(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + updateArgs := &vmUpdateArgs{} + updateArgs.VMClass = vmClass + updateArgs.ResourcePolicy = resourcePolicy + updateArgs.BootstrapData.Data = data + updateArgs.BootstrapData.VAppData = vAppData + updateArgs.BootstrapData.VAppExData = vAppExData + + if res := vmClass.Spec.Policies.Resources; !res.Requests.Cpu.IsZero() || !res.Limits.Cpu.IsZero() { + freq, err := vs.getOrComputeCPUMinFrequency(vmCtx) + if err != nil { + return nil, err + } + updateArgs.MinCPUFreq = freq + } + + var vmClassConfigSpec *types.VirtualMachineConfigSpec + if lib.IsVMClassAsConfigFSSDaynDateEnabled() { + if cs := updateArgs.VMClass.Spec.ConfigSpec; cs != nil { + var err error + vmClassConfigSpec, err = GetVMClassConfigSpec(cs) + if err != nil { + return nil, err + } + } + } + + var vmImageStatus *vmopv1.VirtualMachineImageStatus + // Only get VM image when this is the VM first boot. + if isVMFirstBoot(vmCtx) { + var err error + _, _, vmImageStatus, err = GetVirtualMachineImageSpecAndStatus(vmCtx, vs.k8sClient) + if err != nil { + return nil, err + } + + // The only use of this is for the global JSON_EXTRA_CONFIG to set the image name. + // The global extra config should only be set during first boot. + // TODO: We can just finally kill this with the demise of old gce2e? + renderTemplateFn := func(name, text string) string { + t, err := template.New(name).Parse(text) + if err != nil { + return text + } + b := strings.Builder{} + if err := t.Execute(&b, vmImageStatus); err != nil { + return text + } + return b.String() + } + extraConfig := make(map[string]string, len(vs.globalExtraConfig)) + for k, v := range vs.globalExtraConfig { + extraConfig[k] = renderTemplateFn(k, v) + } + updateArgs.ExtraConfig = extraConfig + + // Enabling the defer-cloud-init extraConfig key for V1Alpha1Compatible images defers cloud-init from running on first boot + // and disables networking configurations by cloud-init. Therefore, only set the extraConfig key to enabled + // when the vmMetadata is nil or when the transport requested is not CloudInit. + // TODO: Is this still actually needed? + updateArgs.VirtualMachineImageV1Alpha1Compatible = + conditions.IsTrueFromConditions(vmImageStatus.Conditions, "VirtualMachineImageV1Alpha1Compatible" /*vmopv1.VirtualMachineImageV1Alpha1CompatibleCondition*/) + } + + updateArgs.ConfigSpec = virtualmachine.CreateConfigSpec( + vmCtx, + vmClassConfigSpec, + &updateArgs.VMClass.Spec, + vmImageStatus, + updateArgs.MinCPUFreq) + + return updateArgs, nil +} + +func isVMFirstBoot(vmCtx context.VirtualMachineContextA2) bool { + if _, ok := vmCtx.VM.Annotations[FirstBootDoneAnnotation]; ok { + return false + } + + return true +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go new file mode 100644 index 000000000..5b516a6cb --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go @@ -0,0 +1,256 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + goctx "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/network" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +// vmE2ETests() tries to close the gap in the existing vmTests() have in the sense that we don't do e2e-like +// tests of the typical VM create/update workflow. This somewhat of a super-set of the vmTests() but those +// tests are already kind of unwieldy and in places, and until we switch over to v1a2, I don't +// to disturb that file so keeping things in sync easier. +// For now, these tests focus on a real - VDS or NSX-T - network env w/ cloud init, and we'll see how these +// need to evolve. +func vmE2ETests() { + + var ( + initObjects []client.Object + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + vmProvider vmprovider.VirtualMachineProviderInterfaceA2 + nsInfo builder.WorkloadNamespaceInfo + + vm *vmopv1.VirtualMachine + vmClass *vmopv1.VirtualMachineClass + ) + + BeforeEach(func() { + // Speed up tests until we Watch the network types. Sigh. + network.RetryTimeout = 1 * time.Millisecond + + testConfig = builder.VCSimTestConfig{ + WithV1A2: true, + WithContentLibrary: true, + WithVMClassAsConfigDaynDate: true, + } + + vm = builder.DummyBasicVirtualMachineA2("test-vm", "") + vmClass = builder.DummyVirtualMachineClassA2() + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + ctx.Context = goctx.WithValue(ctx.Context, context.MaxDeployThreadsContextKey, 1) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + nsInfo = ctx.CreateWorkloadNamespace() + + vmClass.Namespace = nsInfo.Namespace + Expect(ctx.Client.Create(ctx, vmClass)).To(Succeed()) + vmClass.Status.Ready = true + Expect(ctx.Client.Status().Update(ctx, vmClass)).To(Succeed()) + + cloudInitSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cloud-init-secret", + Namespace: nsInfo.Namespace, + }, + StringData: map[string]string{ + "user-value": "", + }, + } + Expect(ctx.Client.Create(ctx, cloudInitSecret)).To(Succeed()) + + vm.Namespace = nsInfo.Namespace + vm.Spec.ClassName = vmClass.Name + vm.Spec.ImageName = ctx.ContentLibraryImageName + vm.Spec.StorageClass = ctx.StorageClassName + vm.Spec.Network.Nameservers = []string{"1.1.1.1", "8.8.8.8"} + vm.Spec.Network.SearchDomains = []string{"vmware.local"} + vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{ + RawCloudConfig: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cloudInitSecret.Name, + }, + }, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + vmProvider = nil + nsInfo = builder.WorkloadNamespaceInfo{} + + vm = nil + vmClass = nil + }) + + Context("VDS", func() { + + const ( + networkName = "my-vds-network" + interfaceName = "eth0" + ) + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvVDS + + vm.Spec.Network.Network = &common.PartialObjectRef{ + Name: networkName, + } + }) + + It("DoIt", func() { + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("simulate successful NetOP reconcile", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.NetworkName).To(Equal(networkName)) + + netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value + netInterface.Status.MacAddress = "" // NetOP doesn't set this. + netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ + { + IP: "192.168.1.110", + IPFamily: netopv1alpha1.IPv4Protocol, + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + err = vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Status.UniqueID).ToNot(BeEmpty()) + vcVM := ctx.GetVMFromMoID(vm.Status.UniqueID) + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + devList, err := vcVM.Device(ctx) + Expect(err).ToNot(HaveOccurred()) + + // For now just check the expected Nic backing. + By("Has expected NIC backing", func() { + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + + dev1 := l[0].GetVirtualDevice() + backingInfo, ok := dev1.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backingInfo.Port.PortgroupKey).To(Equal(ctx.NetworkRef.Reference().Value)) + }) + }) + }) + + Context("NSX-T", func() { + + const ( + networkName = "my-nsxt-network" + interfaceName = "eth0" + ) + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNSXT + + vm.Spec.Network.Network = &common.PartialObjectRef{ + Name: networkName, + } + }) + + It("DoIt", func() { + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + + By("simulate successful NCP reconcile", func() { + netInterface := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NCPCRName(vm.Name, networkName, interfaceName, false), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.VirtualNetwork).To(Equal(networkName)) + + netInterface.Status.MacAddress = "01-23-45-67-89-AB-CD-EF" + netInterface.Status.ProviderStatus = &ncpv1alpha1.VirtualNetworkInterfaceProviderStatus{ + NsxLogicalSwitchID: builder.NsxTLogicalSwitchUUID, + } + netInterface.Status.IPAddresses = []ncpv1alpha1.VirtualNetworkInterfaceIP{ + { + IP: "192.168.1.110", + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + } + netInterface.Status.Conditions = []ncpv1alpha1.VirtualNetworkCondition{ + { + Type: "Ready", + Status: "True", + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + err = vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Status.UniqueID).ToNot(BeEmpty()) + vcVM := ctx.GetVMFromMoID(vm.Status.UniqueID) + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + devList, err := vcVM.Device(ctx) + Expect(err).ToNot(HaveOccurred()) + + // For now just check the expected Nic backing. + By("Has expected NIC backing", func() { + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + + dev1 := l[0].GetVirtualDevice() + backingInfo, ok := dev1.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backingInfo.Port.PortgroupKey).To(Equal(ctx.NetworkRef.Reference().Value)) + }) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go new file mode 100644 index 000000000..fbf325f43 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go @@ -0,0 +1,1813 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "bytes" + goctx "context" + "encoding/json" + "fmt" + "math/rand" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vapi/cluster" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vmTests() { + + const ( + // Hardcoded vcsim CPU frequency. + vcsimCPUFreq = 2294 + + // Default network created for free by vcsim. + dvpgName = "DC0_DVPG0" + ) + + var ( + initObjects []client.Object + testConfig builder.VCSimTestConfig + ctx *builder.TestContextForVCSim + vmProvider vmprovider.VirtualMachineProviderInterfaceA2 + nsInfo builder.WorkloadNamespaceInfo + ) + + BeforeEach(func() { + testConfig = builder.VCSimTestConfig{WithV1A2: true} + }) + + JustBeforeEach(func() { + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) + ctx.Context = goctx.WithValue(ctx.Context, context.MaxDeployThreadsContextKey, 1) + vmProvider = vsphere.NewVSphereVMProviderFromClient(ctx.Client, ctx.Recorder) + nsInfo = ctx.CreateWorkloadNamespace() + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + initObjects = nil + vmProvider = nil + nsInfo = builder.WorkloadNamespaceInfo{} + }) + + Context("Create/Update/Delete VirtualMachine", func() { + var ( + vm *vmopv1.VirtualMachine + vmClass *vmopv1.VirtualMachineClass + ) + + BeforeEach(func() { + testConfig.WithContentLibrary = true + vmClass = builder.DummyVirtualMachineClassA2() + vm = builder.DummyBasicVirtualMachineA2("test-vm", "") + + // Reduce diff from old tests: by default don't create an NIC. + vm.Spec.Network.Disabled = true + }) + + AfterEach(func() { + vmClass = nil + vm = nil + }) + + JustBeforeEach(func() { + vmClass.Namespace = nsInfo.Namespace + Expect(ctx.Client.Create(ctx, vmClass)).To(Succeed()) + vmClass.Status.Ready = true + Expect(ctx.Client.Status().Update(ctx, vmClass)).To(Succeed()) + + clusterVMImage := &vmopv1.ClusterVirtualMachineImage{} + if testConfig.WithContentLibrary { + Expect(ctx.Client.Get(ctx, client.ObjectKey{Name: ctx.ContentLibraryImageName}, clusterVMImage)).To(Succeed()) + } else { + // BMV: VM creation without CL is broken - and has been for a long while - since we assume + // the VM Image will always point to a ContentLibrary item. + // Hack around that with this knob so we can continue to test the VM clone path. + vsphere.SkipVMImageCLProviderCheck = true + + // Use the default VM created by vcsim as the source. + clusterVMImage = builder.DummyClusterVirtualMachineImageA2("DC0_C0_RP0_VM0") + Expect(ctx.Client.Create(ctx, clusterVMImage)).To(Succeed()) + conditions.MarkTrue(clusterVMImage, vmopv1.VirtualMachineImageSyncedCondition) + Expect(ctx.Client.Status().Update(ctx, clusterVMImage)).To(Succeed()) + } + + vm.Namespace = nsInfo.Namespace + vm.Spec.ClassName = vmClass.Name + vm.Spec.ImageName = clusterVMImage.Name + vm.Spec.StorageClass = ctx.StorageClassName + }) + + AfterEach(func() { + vsphere.SkipVMImageCLProviderCheck = false + }) + + createOrUpdateAndGetVcVM := func( + ctx *builder.TestContextForVCSim, + vm *vmopv1.VirtualMachine) (*object.VirtualMachine, error) { + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + if err != nil { + return nil, err + } + + ExpectWithOffset(1, vm.Status.UniqueID).ToNot(BeEmpty()) + vcVM := ctx.GetVMFromMoID(vm.Status.UniqueID) + ExpectWithOffset(1, vcVM).ToNot(BeNil()) + return vcVM, nil + } + + Context("VMClassAsConfigDaynDate FSS is enabled", func() { + + var ( + vcVM *object.VirtualMachine + configSpec *types.VirtualMachineConfigSpec + ethCard types.VirtualEthernetCard + ) + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + testConfig.WithVMClassAsConfigDaynDate = true + + ethCard = types.VirtualEthernetCard{ + VirtualDevice: types.VirtualDevice{ + Key: 4000, + DeviceInfo: &types.Description{ + Label: "test-configspec-nic-label", + Summary: "VM Network", + }, + SlotInfo: &types.VirtualDevicePciBusSlotInfo{ + VirtualDeviceBusSlotInfo: types.VirtualDeviceBusSlotInfo{}, + PciSlotNumber: 32, + }, + ControllerKey: 100, + }, + AddressType: string(types.VirtualEthernetCardMacTypeGenerated), + MacAddress: "00:0c:29:93:d7:27", + ResourceAllocation: &types.VirtualEthernetCardResourceAllocation{ + Reservation: pointer.Int64(42), + }, + } + }) + + JustBeforeEach(func() { + if configSpec != nil { + var w bytes.Buffer + enc := types.NewJSONEncoder(&w) + Expect(enc.Encode(configSpec)).To(Succeed()) + + // Update the VM Class with the XML. + vmClass.Spec.ConfigSpec = w.Bytes() + Expect(ctx.Client.Update(ctx, vmClass)).To(Succeed()) + } + + vm.Spec.Network.Disabled = false + vm.Spec.Network.Network = &common.PartialObjectRef{Name: dvpgName} + + var err error + vcVM, err = createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + vcVM = nil + configSpec = nil + }) + + Context("VM Class has no ConfigSpec", func() { + BeforeEach(func() { + configSpec = nil + }) + + It("creates VM", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + }) + + Context("ConfigSpec specifies hardware spec", func() { + BeforeEach(func() { + configSpec = &types.VirtualMachineConfigSpec{ + Name: "config-spec-name-is-not-used", + NumCPUs: 7, + MemoryMB: 5102, + } + }) + + It("CPU and memory from ConfigSpec are ignored", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Summary.Config.Name).To(Equal(vm.Name)) + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.NumCpu).ToNot(BeEquivalentTo(configSpec.NumCPUs)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + Expect(o.Summary.Config.MemorySizeMB).ToNot(BeEquivalentTo(configSpec.MemoryMB)) + }) + }) + + Context("VM Class spec CPU reservation & limits are non-zero and ConfigSpec specifies CPU reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Cpu = resource.MustParse("2") + vmClass.Spec.Policies.Resources.Limits.Cpu = resource.MustParse("3") + + // Specify a CPU reservation via ConfigSpec. This value should not be honored. + configSpec = &types.VirtualMachineConfigSpec{ + CpuAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(6), + }, + } + }) + + It("VM gets CPU reservation from VM Class spec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + resources := &vmClass.Spec.Policies.Resources + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.CpuAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.CPUQuantityToMhz(resources.Requests.Cpu, vcsimCPUFreq))) + Expect(*reservation).ToNot(Equal(*configSpec.CpuAllocation.Reservation)) + + limit := o.Config.CpuAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.CPUQuantityToMhz(resources.Limits.Cpu, vcsimCPUFreq))) + }) + }) + + Context("VM Class spec CPU reservation is zero and ConfigSpec specifies CPU reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Cpu = resource.MustParse("0") + vmClass.Spec.Policies.Resources.Limits.Cpu = resource.MustParse("0") + + // Specify a CPU reservation via ConfigSpec + configSpec = &types.VirtualMachineConfigSpec{ + CpuAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(6), + }, + } + }) + + It("VM gets CPU reservation from ConfigSpec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.CpuAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).ToNot(BeZero()) + Expect(*reservation).To(Equal(*configSpec.CpuAllocation.Reservation)) + }) + }) + + Context("VM Class spec Memory reservation & limits are non-zero and ConfigSpec specifies memory reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Memory = resource.MustParse("4Mi") + vmClass.Spec.Policies.Resources.Limits.Memory = resource.MustParse("4Mi") + + // Specify a Memory reservation via ConfigSpec + configSpec = &types.VirtualMachineConfigSpec{ + MemoryAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(5120), + }, + } + }) + + It("VM gets memory reservation from VM Class spec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + resources := &vmClass.Spec.Policies.Resources + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.MemoryAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.MemoryQuantityToMb(resources.Requests.Memory))) + Expect(*reservation).ToNot(Equal(*configSpec.MemoryAllocation.Reservation)) + + limit := o.Config.MemoryAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.MemoryQuantityToMb(resources.Limits.Memory))) + }) + }) + + Context("VM Class spec Memory reservations are zero and ConfigSpec specifies memory reservation", func() { + BeforeEach(func() { + vmClass.Spec.Policies.Resources.Requests.Memory = resource.MustParse("0Mi") + vmClass.Spec.Policies.Resources.Limits.Memory = resource.MustParse("0Mi") + + // Specify a Memory reservation via ConfigSpec + configSpec = &types.VirtualMachineConfigSpec{ + MemoryAllocation: &types.ResourceAllocationInfo{ + Reservation: pointer.Int64(5120), + }, + } + }) + + It("VM gets memory reservation from ConfigSpec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + reservation := o.Config.MemoryAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).ToNot(BeZero()) + Expect(*reservation).To(Equal(*configSpec.MemoryAllocation.Reservation)) + }) + }) + + Context("VM Class ConfigSpec specifies a network interface", func() { + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + + // Create the ConfigSpec with an ethernet card. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualE1000{ + VirtualEthernetCard: ethCard, + }, + }, + }, + } + }) + + // FIXME: Has extra NIC b/c of vcsim DeployOVF bug + It("Reconfigures the VM with the NIC specified in ConfigSpec", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + // Expect(l).To(HaveLen(1 + 1)) + + dev := l[0].GetVirtualDevice() + // dev := l[0+1].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + + ethDevice, ok := l[0].(*types.VirtualE1000) + // ethDevice, ok := l[0+1].(*types.VirtualE1000) + Expect(ok).To(BeTrue()) + Expect(ethDevice.AddressType).To(Equal(ethCard.AddressType)) + Expect(ethDevice.MacAddress).To(Equal(ethCard.MacAddress)) + + Expect(dev.DeviceInfo).To(Equal(ethCard.VirtualDevice.DeviceInfo)) + Expect(dev.DeviceGroupInfo).To(Equal(ethCard.VirtualDevice.DeviceGroupInfo)) + Expect(dev.SlotInfo).To(Equal(ethCard.VirtualDevice.SlotInfo)) + Expect(dev.ControllerKey).To(Equal(ethCard.VirtualDevice.ControllerKey)) + Expect(ethDevice.ResourceAllocation).ToNot(BeNil()) + Expect(ethDevice.ResourceAllocation.Reservation).ToNot(BeNil()) + Expect(*ethDevice.ResourceAllocation.Reservation).To(Equal(*ethCard.ResourceAllocation.Reservation)) + }) + }) + + Context("ConfigSpec does not specify any network interfaces", func() { + + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + + configSpec = &types.VirtualMachineConfigSpec{} + }) + + // FIXME: Has extra NIC b/c of vcsim DeployOVF bug + It("Reconfigures the VM with the default NIC settings from provider", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + // Expect(l).To(HaveLen(1 + 1)) + + dev := l[0].GetVirtualDevice() + // dev := l[0+1].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + }) + }) + + Context("VM Class Spec and ConfigSpec both contain GPU and DirectPath devices", func() { + BeforeEach(func() { + vmClass.Spec.Hardware.Devices = vmopv1.VirtualDevices{ + VGPUDevices: []vmopv1.VGPUDevice{ + { + ProfileName: "profile-from-class", + }, + }, + DynamicDirectPathIODevices: []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 50, + DeviceID: 51, + CustomLabel: "label-from-class", + }, + }, + } + + // Create the ConfigSpec with a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-config-spec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-config-spec", + }, + }, + }, + }, + }, + } + }) + + It("GPU and DirectPath devices from VM Class Spec.Devices are ignored", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + p := devList.SelectByType(&types.VirtualPCIPassthrough{}) + Expect(p).To(HaveLen(2)) + + pciDev1 := p[0].GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).Should(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal("profile-from-config-spec")) + + pciDev2 := p[1].GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*types.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).Should(BeTrue()) + Expect(pciBacking2.AllowedDevice).To(HaveLen(1)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(Equal(int32(52))) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(Equal(int32(53))) + Expect(pciBacking2.CustomLabel).To(Equal("label-from-config-spec")) + }) + }) + + Context("VM Class Config specifies an ethCard, a GPU and a DDPIO device", func() { + + BeforeEach(func() { + // Create the ConfigSpec with an ethernet card, a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualE1000{ + VirtualEthernetCard: ethCard, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "SampleProfile2", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "SampleLabel2", + }, + }, + }, + }, + }, + } + }) + + // FIXME: Has extra NIC b/c of vcsim DeployOVF bug + It("Reconfigures the VM with a NIC, GPU and DDPIO device specified in ConfigSpec", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(1)) + // Expect(l).To(HaveLen(1 + 1)) + + dev := l[0].GetVirtualDevice() + // dev := l[0+1].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + + ethDevice, ok := l[0].(*types.VirtualE1000) + // ethDevice, ok := l[0+1].(*types.VirtualE1000) + Expect(ok).To(BeTrue()) + Expect(ethDevice.AddressType).To(Equal(ethCard.AddressType)) + Expect(dev.DeviceInfo).To(Equal(ethCard.VirtualDevice.DeviceInfo)) + Expect(dev.DeviceGroupInfo).To(Equal(ethCard.VirtualDevice.DeviceGroupInfo)) + Expect(dev.SlotInfo).To(Equal(ethCard.VirtualDevice.SlotInfo)) + Expect(dev.ControllerKey).To(Equal(ethCard.VirtualDevice.ControllerKey)) + Expect(ethDevice.MacAddress).To(Equal(ethCard.MacAddress)) + Expect(ethDevice.ResourceAllocation).ToNot(BeNil()) + Expect(ethDevice.ResourceAllocation.Reservation).ToNot(BeNil()) + Expect(*ethDevice.ResourceAllocation.Reservation).To(Equal(*ethCard.ResourceAllocation.Reservation)) + + p := devList.SelectByType(&types.VirtualPCIPassthrough{}) + Expect(p).To(HaveLen(2)) + pciDev1 := p[0].GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).Should(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal("SampleProfile2")) + pciDev2 := p[1].GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*types.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).Should(BeTrue()) + Expect(pciBacking2.AllowedDevice).To(HaveLen(1)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(Equal(int32(52))) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(Equal(int32(53))) + Expect(pciBacking2.CustomLabel).To(Equal("SampleLabel2")) + + // CPU and memory should be from vm class + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + }) + + Context("VM Class Config specifies disks, disk controllers, other miscellaneous devices", func() { + BeforeEach(func() { + // Create the ConfigSpec with disks, disk controller and some misc devices: pointing device, + // video card, etc. This works fine with vcsim and helps with testing adding misc devices. + // The simulator can still reconfigure the VM with default device types like pointing devices, + // keyboard, video card, etc. But VC has some restrictions with reconfiguring a VM with new + // default device types via ConfigSpec and are usually ignored. + configSpec = &types.VirtualMachineConfigSpec{ + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPointingDevice{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPointingDeviceDeviceBackingInfo{ + HostPointingDevice: "autodetect", + }, + Key: 700, + ControllerKey: 300, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPS2Controller{ + VirtualController: types.VirtualController{ + Device: []int32{700}, + VirtualDevice: types.VirtualDevice{ + Key: 300, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualMachineVideoCard{ + UseAutoDetect: pointer.Bool(false), + NumDisplays: 1, + VirtualDevice: types.VirtualDevice{ + Key: 500, + ControllerKey: 100, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIController{ + VirtualController: types.VirtualController{ + Device: []int32{500}, + VirtualDevice: types.VirtualDevice{ + Key: 100, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualDisk{ + CapacityInBytes: 1024, + VirtualDevice: types.VirtualDevice{ + Key: -42, + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(true), + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualSCSIController{ + VirtualController: types.VirtualController{ + Device: []int32{-42}, + }, + }, + }, + }, + } + }) + + // FIXME: vcsim behavior needs to be closer to real VC here so there aren't dupes + It("Reconfigures the VM with all misc devices in ConfigSpec except disk and disk controllers", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + + // VM already has a default pointing device and the spec adds one more + // info about the default device is unknown to assert on + pointingDev := devList.SelectByType(&types.VirtualPointingDevice{}) + Expect(pointingDev).To(HaveLen(2)) + dev := pointingDev[0].GetVirtualDevice() + backing, ok := dev.Backing.(*types.VirtualPointingDeviceDeviceBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backing.HostPointingDevice).To(Equal("autodetect")) + Expect(dev.Key).To(Equal(int32(700))) + Expect(dev.ControllerKey).To(Equal(int32(300))) + + ps2Controllers := devList.SelectByType(&types.VirtualPS2Controller{}) + Expect(ps2Controllers).To(HaveLen(1)) + dev = ps2Controllers[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(300))) + + pciControllers := devList.SelectByType(&types.VirtualPCIController{}) + Expect(pciControllers).To(HaveLen(1)) + dev = pciControllers[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(100))) + + // VM already has a default video card and the spec adds one more + // info about the default device is unknown to assert on + video := devList.SelectByType(&types.VirtualMachineVideoCard{}) + Expect(video).To(HaveLen(2)) + dev = video[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(500))) + Expect(dev.ControllerKey).To(Equal(int32(100))) + + // Disk and disk controllers from config spec should not get added, since we + // filter them out in our ConfigSpec + diskControllers := devList.SelectByType(&types.VirtualSCSIController{}) + Expect(diskControllers).To(BeEmpty()) + + // Only preexisting disk should be present on VM -- len: 1 + disks := devList.SelectByType(&types.VirtualDisk{}) + Expect(disks).To(HaveLen(1)) + dev = disks[0].GetVirtualDevice() + Expect(dev.Key).ToNot(Equal(int32(-42))) + }) + }) + + Context("VM Class Config does not specify a hardware version", func() { + + Context("VM Class has vGPU and/or DDPIO devices", func() { + BeforeEach(func() { + // Create the ConfigSpec with a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-configspec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-configspec", + }, + }, + }, + }, + }, + } + }) + + It("creates a VM with a hardware version minimum supported for PCI devices", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPCIPassthruDevices))) + }) + }) + + Context("VM Class has vGPU and/or DDPIO devices and VM spec has a PVC", func() { + BeforeEach(func() { + // Create the ConfigSpec with a GPU and a DDPIO device. + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-configspec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-configspec", + }, + }, + }, + }, + }, + } + + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: "dummy-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-claim-1", + }, + }, + }, + }, + } + + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: "dummy-vol", + Attached: true, + }, + } + }) + + It("creates a VM with a hardware version minimum supported for PCI devices", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPCIPassthruDevices))) + }) + }) + + Context("VM spec has a PVC", func() { + BeforeEach(func() { + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: "dummy-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-claim-1", + }, + }, + }, + }, + } + + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: "dummy-vol", + Attached: true, + }, + } + }) + + It("creates a VM with a hardware version minimum supported for PVCs", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPVC))) + }) + }) + }) + + Context("VMClassAsConfig FSS is Enabled", func() { + + BeforeEach(func() { + testConfig.WithVMClassAsConfig = true + }) + + When("configSpec has disk and disk controllers", func() { + BeforeEach(func() { + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualSATAController{ + VirtualController: types.VirtualController{ + VirtualDevice: types.VirtualDevice{ + Key: 101, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualSCSIController{ + VirtualController: types.VirtualController{ + VirtualDevice: types.VirtualDevice{ + Key: 103, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualNVMEController{ + VirtualController: types.VirtualController{ + VirtualDevice: types.VirtualDevice{ + Key: 104, + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualDisk{ + CapacityInBytes: 1024, + VirtualDevice: types.VirtualDevice{ + Key: -42, + Backing: &types.VirtualDiskFlatVer2BackingInfo{ + ThinProvisioned: pointer.Bool(true), + }, + }, + }, + }, + }, + } + }) + + It("creates a VM with disk controllers", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + satacont := devList.SelectByType(&types.VirtualSATAController{}) + Expect(satacont).To(HaveLen(1)) + dev := satacont[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(101))) + + scsicont := devList.SelectByType(&types.VirtualSCSIController{}) + Expect(scsicont).To(HaveLen(1)) + dev = scsicont[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(103))) + + nvmecont := devList.SelectByType(&types.VirtualNVMEController{}) + Expect(nvmecont).To(HaveLen(1)) + dev = nvmecont[0].GetVirtualDevice() + Expect(dev.Key).To(Equal(int32(104))) + + // only preexisting disk should be present on VM -- len: 1 + disks := devList.SelectByType(&types.VirtualDisk{}) + Expect(disks).To(HaveLen(1)) + dev1 := disks[0].GetVirtualDevice() + Expect(dev1.Key).ToNot(Equal(int32(-42))) + }) + }) + }) + }) + + Context("CreateOrUpdate VM", func() { + + It("Basic VM", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + By("has expected Status values", func() { + Expect(vm.Status.PowerState).To(Equal(vm.Spec.PowerState)) + Expect(vm.Status.Host).ToNot(BeEmpty()) + Expect(vm.Status.InstanceUUID).To(And(Not(BeEmpty()), Equal(o.Config.InstanceUuid))) + Expect(vm.Status.BiosUUID).To(And(Not(BeEmpty()), Equal(o.Config.Uuid))) + + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionClassReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionStorageReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + }) + + By("has expected inventory path", func() { + Expect(vcVM.InventoryPath).To(HaveSuffix(fmt.Sprintf("/%s/%s", nsInfo.Namespace, vm.Name))) + }) + + By("has expected namespace resource pool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + + By("has expected power state", func() { + Expect(o.Summary.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + }) + + vmClassRes := &vmClass.Spec.Policies.Resources + + By("has expected CpuAllocation", func() { + Expect(o.Config.CpuAllocation).ToNot(BeNil()) + + reservation := o.Config.CpuAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.CPUQuantityToMhz(vmClassRes.Requests.Cpu, vcsimCPUFreq))) + limit := o.Config.CpuAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.CPUQuantityToMhz(vmClassRes.Limits.Cpu, vcsimCPUFreq))) + }) + + By("has expected MemoryAllocation", func() { + Expect(o.Config.MemoryAllocation).ToNot(BeNil()) + + reservation := o.Config.MemoryAllocation.Reservation + Expect(reservation).ToNot(BeNil()) + Expect(*reservation).To(Equal(virtualmachine.MemoryQuantityToMb(vmClassRes.Requests.Memory))) + limit := o.Config.MemoryAllocation.Limit + Expect(limit).ToNot(BeNil()) + Expect(*limit).To(Equal(virtualmachine.MemoryQuantityToMb(vmClassRes.Limits.Memory))) + }) + + By("has expected hardware config", func() { + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + + // TODO: More assertions! + }) + + Context("VM Class with PCI passthrough devices", func() { + BeforeEach(func() { + vmClass.Spec.Hardware.Devices = vmopv1.VirtualDevices{ + VGPUDevices: []vmopv1.VGPUDevice{ + { + ProfileName: "profile-from-class-without-class-as-config-fss", + }, + }, + DynamicDirectPathIODevices: []vmopv1.DynamicDirectPathIODevice{ + { + VendorID: 59, + DeviceID: 60, + CustomLabel: "label-from-class-without-class-as-config-fss", + }, + }, + } + }) + + It("VM should have expected PCI devices from VM Class", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + p := devList.SelectByType(&types.VirtualPCIPassthrough{}) + Expect(p).To(HaveLen(2)) + + pciDev1 := p[0].GetVirtualDevice() + pciBacking1, ok1 := pciDev1.Backing.(*types.VirtualPCIPassthroughVmiopBackingInfo) + Expect(ok1).Should(BeTrue()) + Expect(pciBacking1.Vgpu).To(Equal("profile-from-class-without-class-as-config-fss")) + + pciDev2 := p[1].GetVirtualDevice() + pciBacking2, ok2 := pciDev2.Backing.(*types.VirtualPCIPassthroughDynamicBackingInfo) + Expect(ok2).Should(BeTrue()) + Expect(pciBacking2.AllowedDevice).To(HaveLen(1)) + Expect(pciBacking2.AllowedDevice[0].VendorId).To(Equal(int32(59))) + Expect(pciBacking2.AllowedDevice[0].DeviceId).To(Equal(int32(60))) + Expect(pciBacking2.CustomLabel).To(Equal("label-from-class-without-class-as-config-fss")) + }) + }) + + Context("Without Storage Class", func() { + BeforeEach(func() { + testConfig.WithoutStorageClass = true + }) + + It("Creates VM", func() { + Expect(vm.Spec.StorageClass).To(BeEmpty()) + + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + By("has expected datastore", func() { + datastore, err := ctx.Finder.DefaultDatastore(ctx) + Expect(err).ToNot(HaveOccurred()) + + Expect(o.Datastore).To(HaveLen(1)) + Expect(o.Datastore[0]).To(Equal(datastore.Reference())) + }) + }) + }) + + Context("Without Content Library", func() { + BeforeEach(func() { + testConfig.WithContentLibrary = false + }) + + // TODO: Dedupe this with "Basic VM" above + It("Clones VM", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + By("has expected Status values", func() { + Expect(vm.Status.PowerState).To(Equal(vm.Spec.PowerState)) + Expect(vm.Status.Host).ToNot(BeEmpty()) + Expect(vm.Status.InstanceUUID).To(And(Not(BeEmpty()), Equal(o.Config.InstanceUuid))) + Expect(vm.Status.BiosUUID).To(And(Not(BeEmpty()), Equal(o.Config.Uuid))) + + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionClassReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionStorageReady)).To(BeTrue()) + + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + }) + + By("has expected inventory path", func() { + Expect(vcVM.InventoryPath).To(HaveSuffix(fmt.Sprintf("/%s/%s", nsInfo.Namespace, vm.Name))) + }) + + By("has expected namespace resource pool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + + By("has expected power state", func() { + Expect(o.Summary.Runtime.PowerState).To(Equal(types.VirtualMachinePowerStatePoweredOn)) + }) + + By("has expected hardware config", func() { + // TODO: Fix vcsim behavior: NumCPU is correct "2" in the CloneSpec.Config but ends up + // with 1 CPU from source VM. Ditto for MemorySize. These assertions are only working + // because the state is on so we reconfigure the VM after it is created. + Expect(o.Summary.Config.NumCpu).To(BeEquivalentTo(vmClass.Spec.Hardware.Cpus)) + Expect(o.Summary.Config.MemorySizeMB).To(BeEquivalentTo(vmClass.Spec.Hardware.Memory.Value() / 1024 / 1024)) + }) + + // TODO: More assertions! + }) + }) + + // BMV: I don't think this is actually supported. + XIt("Create VM from VMTX in ContentLibrary", func() { + imageName := "test-vm-vmtx" + + ctx.ContentLibraryItemTemplate("DC0_C0_RP0_VM0", imageName) + vm.Spec.ImageName = imageName + + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("When fault domains is enabled", func() { + BeforeEach(func() { + testConfig.WithFaultDomains = true + }) + + It("creates VM in placement selected zone", func() { + Expect(vm.Labels).ToNot(HaveKey(topology.KubernetesTopologyZoneLabelKey)) + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + azName, ok := vm.Labels[topology.KubernetesTopologyZoneLabelKey] + Expect(ok).To(BeTrue()) + Expect(azName).To(BeElementOf(ctx.ZoneNames)) + + By("VM is created in the zone's ResourcePool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, azName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + }) + + It("creates VM in assigned zone", func() { + azName := ctx.ZoneNames[rand.Intn(len(ctx.ZoneNames))] //nolint:gosec + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = azName + + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + By("VM is created in the zone's ResourcePool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + nsRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, azName, "") + Expect(nsRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(nsRP.Reference().Value)) + }) + }) + }) + + Context("When Instance Storage FSS is enabled", func() { + BeforeEach(func() { + testConfig.WithInstanceStorage = true + }) + + expectInstanceStorageVolumes := func( + vm *vmopv1.VirtualMachine, + isStorage vmopv1.InstanceStorage) { + + ExpectWithOffset(1, isStorage.Volumes).ToNot(BeEmpty()) + isVolumes := instancestorage.FilterVolumes(vm) + ExpectWithOffset(1, isVolumes).To(HaveLen(len(isStorage.Volumes))) + + for _, isVol := range isStorage.Volumes { + found := false + + for idx, vol := range isVolumes { + claim := vol.PersistentVolumeClaim.InstanceVolumeClaim + if claim.StorageClass == isStorage.StorageClass && claim.Size == isVol.Size { + isVolumes = append(isVolumes[:idx], isVolumes[idx+1:]...) + found = true + break + } + } + + ExpectWithOffset(1, found).To(BeTrue(), "failed to find instance storage volume for %v", isVol) + } + } + + It("creates VM without instance storage", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + }) + + It("create VM with instance storage", func() { + Expect(vm.Spec.Volumes).To(BeEmpty()) + + vmClass.Spec.Hardware.InstanceStorage = vmopv1.InstanceStorage{ + StorageClass: vm.Spec.StorageClass, + Volumes: []vmopv1.InstanceStorageVolume{ + { + Size: resource.MustParse("256Gi"), + }, + { + Size: resource.MustParse("512Gi"), + }, + }, + } + Expect(ctx.Client.Update(ctx, vmClass)).To(Succeed()) + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(MatchError("instance storage PVCs are not bound yet")) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeFalse()) + + By("Instance storage volumes should be added to VM", func() { + Expect(instancestorage.IsPresent(vm)).To(BeTrue()) + expectInstanceStorageVolumes(vm, vmClass.Spec.Hardware.InstanceStorage) + }) + + By("Placement should have been done", func() { + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionPlacementReady)).To(BeTrue()) + Expect(vm.Annotations).To(HaveKey(constants.InstanceStorageSelectedNodeAnnotationKey)) + Expect(vm.Annotations).To(HaveKey(constants.InstanceStorageSelectedNodeMOIDAnnotationKey)) + }) + + isVol0 := vm.Spec.Volumes[0] + Expect(isVol0.PersistentVolumeClaim.InstanceVolumeClaim).ToNot(BeNil()) + + By("simulate volume controller workflow", func() { + // Simulate what would be set by volume controller. + vm.Annotations[constants.InstanceStoragePVCsBoundAnnotationKey] = "" + + err = vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("status update pending for persistent volume: %s on VM", isVol0.Name))) + + // Simulate what would be set by the volume controller. + for _, vol := range vm.Spec.Volumes { + vm.Status.Volumes = append(vm.Status.Volumes, vmopv1.VirtualMachineVolumeStatus{ + Name: vol.Name, + Attached: true, + }) + } + }) + + By("VM is now created", func() { + _, err = createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) + }) + }) + }) + + It("Powers VM off", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOn)) + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + state, err := vcVM.PowerState(ctx) + Expect(err).ToNot(HaveOccurred()) + Expect(state).To(Equal(types.VirtualMachinePowerStatePoweredOff)) + }) + + It("returns error when StorageClass is required but none specified", func() { + vm.Spec.StorageClass = "" + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(MatchError("StorageClass is required but not specified")) + }) + + It("Can be called multiple times", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + modified := o.Config.Modified + + _, err = createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + // Try to assert nothing changed. + Expect(o.Config.Modified).To(Equal(modified)) + }) + + Context("VM Metadata", func() { + + Context("ExtraConfig Transport", func() { + var ec map[string]interface{} + + JustBeforeEach(func() { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "md-configmap-", + Namespace: vm.Namespace, + }, + Data: map[string]string{ + "foo.bar": "should-be-ignored", + "guestinfo.Foo": "foo", + }, + } + Expect(ctx.Client.Create(ctx, configMap)).To(Succeed()) + + /* + vm.Spec.VmMetadata = &vmopv1.VirtualMachineMetadata{ + ConfigMapName: configMap.Name, + Transport: vmopv1.VirtualMachineMetadataExtraConfigTransport, + } + */ + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + ec = map[string]interface{}{} + for _, option := range o.Config.ExtraConfig { + if val := option.GetOptionValue(); val != nil { + ec[val.Key] = val.Value.(string) + } + } + }) + + AfterEach(func() { + ec = nil + }) + + // TODO: As is we can't really honor "guestinfo.*" prefix + XIt("Metadata data is included in ExtraConfig", func() { + Expect(ec).ToNot(HaveKey("foo.bar")) + Expect(ec).To(HaveKeyWithValue("guestinfo.Foo", "foo")) + + By("Should include default keys and values", func() { + Expect(ec).To(HaveKeyWithValue("disk.enableUUID", "TRUE")) + Expect(ec).To(HaveKeyWithValue("vmware.tools.gosc.ignoretoolscheck", "TRUE")) + }) + }) + + Context("JSON_EXTRA_CONFIG is specified", func() { + BeforeEach(func() { + b, err := json.Marshal( + struct { + Foo string + Bar string + }{ + Foo: "f00", + Bar: "42", + }, + ) + Expect(err).ToNot(HaveOccurred()) + testConfig.WithJSONExtraConfig = string(b) + }) + + It("Global config is included in ExtraConfig", func() { + Expect(ec).To(HaveKeyWithValue("Foo", "f00")) + Expect(ec).To(HaveKeyWithValue("Bar", "42")) + }) + }) + }) + }) + + Context("Network", func() { + + It("Should not have a nic", func() { + Expect(vm.Spec.Network.Disabled).To(BeTrue()) + + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(BeEmpty()) + }) + + Context("Multiple NICs are specified", func() { + BeforeEach(func() { + testConfig.WithNetworkEnv = builder.NetworkEnvNamed + + vm.Spec.Network.Disabled = false + vm.Spec.Network.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: "VM Network"}, + }, + { + Name: "eth1", + Network: common.PartialObjectRef{Name: dvpgName}, + }, + } + }) + + It("Has expected devices", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByType(&types.VirtualEthernetCard{}) + Expect(l).To(HaveLen(2)) + + dev1 := l[0].GetVirtualDevice() + backing1, ok := dev1.Backing.(*types.VirtualEthernetCardNetworkBackingInfo) + Expect(ok).Should(BeTrue()) + Expect(backing1.DeviceName).To(Equal("VM Network")) + + dev2 := l[1].GetVirtualDevice() + backing2, ok := dev2.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) + Expect(ok).Should(BeTrue()) + _, dvpg := getDVPG(ctx, dvpgName) + Expect(backing2.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) + }) + }) + }) + + Context("Disks", func() { + + Context("VM has thin provisioning", func() { + BeforeEach(func() { + vm.Spec.Advanced.DefaultVolumeProvisioningMode = vmopv1.VirtualMachineVolumeProvisioningModeThin + }) + + It("Succeeds", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + _, backing := getVMHomeDisk(ctx, vcVM, o) + Expect(backing.ThinProvisioned).To(PointTo(BeTrue())) + }) + }) + + XContext("VM has thick provisioning", func() { + BeforeEach(func() { + vm.Spec.Advanced.DefaultVolumeProvisioningMode = vmopv1.VirtualMachineVolumeProvisioningModeThick + }) + + It("Succeeds", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + /* vcsim CL deploy has "thick" but that isn't reflected for this disk. */ + _, backing := getVMHomeDisk(ctx, vcVM, o) + Expect(backing.ThinProvisioned).To(PointTo(BeFalse())) + }) + }) + + XContext("VM has eager zero provisioning", func() { + BeforeEach(func() { + vm.Spec.Advanced.DefaultVolumeProvisioningMode = vmopv1.VirtualMachineVolumeProvisioningModeThickEagerZero + }) + + It("Succeeds", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + + /* vcsim CL deploy has "eagerZeroedThick" but that isn't reflected for this disk. */ + _, backing := getVMHomeDisk(ctx, vcVM, o) + Expect(backing.EagerlyScrub).To(PointTo(BeTrue())) + }) + }) + + Context("Should resize root disk", func() { + newSize := resource.MustParse("4242Gi") + + It("Succeeds", func() { + vm.Spec.Advanced.BootDiskCapacity = newSize + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOn + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + disk, _ := getVMHomeDisk(ctx, vcVM, o) + Expect(disk.CapacityInBytes).To(BeEquivalentTo(newSize.Value())) + }) + }) + }) + + Context("CNS Volumes", func() { + cnsVolumeName := "cns-volume-1" + + It("CSI Volumes workflow", func() { + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOn + By("Add CNS volume to VM", func() { + vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: cnsVolumeName, + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-volume-1", + }, + }, + }, + }, + } + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("status update pending for persistent volume: %s on VM", cnsVolumeName))) + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + }) + + By("CNS volume is not attached", func() { + errMsg := "blah blah blah not attached" + + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: cnsVolumeName, + Attached: false, + Error: errMsg, + }, + } + + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(fmt.Sprintf("persistent volume: %s not attached to VM", cnsVolumeName))) + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + }) + + By("CNS volume is attached", func() { + vm.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: cnsVolumeName, + Attached: true, + }, + } + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOn)) + }) + }) + }) + + Context("When fault domains is enabled", func() { + const zoneName = "az-1" + + BeforeEach(func() { + testConfig.WithFaultDomains = true + // Explicitly place the VM into one of the zones that the test context will create. + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + }) + + It("Reverse lookups existing VM into correct zone", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + Expect(vm.Status.Zone).To(Equal(zoneName)) + delete(vm.Labels, topology.KubernetesTopologyZoneLabelKey) + + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + Expect(vm.Status.Zone).To(Equal(zoneName)) + }) + }) + }) + + Context("VM SetResourcePolicy", func() { + var resourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + + JustBeforeEach(func() { + resourcePolicyName := "test-policy" + resourcePolicy = getVirtualMachineSetResourcePolicy(resourcePolicyName, nsInfo.Namespace) + Expect(vmProvider.CreateOrUpdateVirtualMachineSetResourcePolicy(ctx, resourcePolicy)).To(Succeed()) + Expect(ctx.Client.Create(ctx, resourcePolicy)).To(Succeed()) + + vm.Annotations["vsphere-cluster-module-group"] = resourcePolicy.Spec.ClusterModuleGroups[0] + vm.Spec.Reserved.ResourcePolicyName = resourcePolicy.Name + }) + + AfterEach(func() { + resourcePolicy = nil + }) + + It("VM is created in child Folder and ResourcePool", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + By("has expected inventory path", func() { + Expect(vcVM.InventoryPath).To(HaveSuffix( + fmt.Sprintf("/%s/%s/%s", nsInfo.Namespace, resourcePolicy.Spec.Folder, vm.Name))) + }) + + By("has expected namespace resource pool", func() { + rp, err := vcVM.ResourcePool(ctx) + Expect(err).ToNot(HaveOccurred()) + childRP := ctx.GetResourcePoolForNamespace(nsInfo.Namespace, "", resourcePolicy.Spec.ResourcePool.Name) + Expect(childRP).ToNot(BeNil()) + Expect(rp.Reference().Value).To(Equal(childRP.Reference().Value)) + }) + }) + + It("Cluster Modules", func() { + vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + members, err := cluster.NewManager(ctx.RestClient).ListModuleMembers(ctx, resourcePolicy.Status.ClusterModules[0].ModuleUuid) + Expect(err).ToNot(HaveOccurred()) + Expect(members).To(ContainElements(vcVM.Reference())) + }) + + It("Returns error with non-existence cluster module", func() { + vm.Annotations["vsphere-cluster-module-group"] = "bogusClusterMod" + err := vmProvider.CreateOrUpdateVirtualMachine(ctx, vm) + Expect(err).To(MatchError("ClusterModule bogusClusterMod not found")) + }) + }) + + Context("Delete VM", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + Context("when the VM is off", func() { + BeforeEach(func() { + vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff + }) + + It("deletes the VM", func() { + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOff)) + + uniqueID := vm.Status.UniqueID + Expect(ctx.GetVMFromMoID(uniqueID)).ToNot(BeNil()) + + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(ctx.GetVMFromMoID(uniqueID)).To(BeNil()) + }) + }) + + It("when the VM is on", func() { + Expect(vm.Status.PowerState).To(Equal(vmopv1.VirtualMachinePowerStateOn)) + + uniqueID := vm.Status.UniqueID + Expect(ctx.GetVMFromMoID(uniqueID)).ToNot(BeNil()) + + // This checks that we power off the VM prior to deletion. + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(ctx.GetVMFromMoID(uniqueID)).To(BeNil()) + }) + + It("returns success when VM does not exist", func() { + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + }) + + Context("When fault domains is enabled", func() { + const zoneName = "az-1" + + BeforeEach(func() { + testConfig.WithFaultDomains = true + // Explicitly place the VM into one of the zones that the test context will create. + vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName + }) + + It("returns NotFound when VM does not exist", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + delete(vm.Labels, topology.KubernetesTopologyZoneLabelKey) + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("Deletes existing VM when zone info is missing", func() { + _, err := createOrUpdateAndGetVcVM(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + + uniqueID := vm.Status.UniqueID + Expect(ctx.GetVMFromMoID(uniqueID)).ToNot(BeNil()) + + Expect(vm.Labels).To(HaveKeyWithValue(topology.KubernetesTopologyZoneLabelKey, zoneName)) + delete(vm.Labels, topology.KubernetesTopologyZoneLabelKey) + + Expect(vmProvider.DeleteVirtualMachine(ctx, vm)).To(Succeed()) + Expect(ctx.GetVMFromMoID(uniqueID)).To(BeNil()) + }) + }) + }) + + Context("Guest Heartbeat", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("return guest heartbeat", func() { + heartbeat, err := vmProvider.GetVirtualMachineGuestHeartbeat(ctx, vm) + Expect(err).ToNot(HaveOccurred()) + // Just testing for property query: field not set in vcsim. + Expect(heartbeat).To(BeEmpty()) + }) + }) + + Context("Web console ticket", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("return ticket", func() { + // vcsim doesn't implement this yet so expect an error. + _, err := vmProvider.GetVirtualMachineWebMKSTicket(ctx, vm, "foo") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not implement: AcquireTicket")) + }) + }) + + Context("VM hardware version", func() { + JustBeforeEach(func() { + Expect(vmProvider.CreateOrUpdateVirtualMachine(ctx, vm)).To(Succeed()) + }) + + It("return version", func() { + version, err := vmProvider.GetVirtualMachineHardwareVersion(ctx, vm) + Expect(err).NotTo(HaveOccurred()) + Expect(version).To(Equal(int32(9))) + }) + }) + }) +} + +// getVMHomeDisk gets the VM's "home" disk. It makes some assumptions about the backing and disk name. +func getVMHomeDisk( + ctx *builder.TestContextForVCSim, + vcVM *object.VirtualMachine, + o mo.VirtualMachine) (*types.VirtualDisk, *types.VirtualDiskFlatVer2BackingInfo) { + + ExpectWithOffset(1, vcVM.Name()).ToNot(BeEmpty()) + ExpectWithOffset(1, o.Datastore).ToNot(BeEmpty()) + var dso mo.Datastore + ExpectWithOffset(1, vcVM.Properties(ctx, o.Datastore[0], nil, &dso)).To(Succeed()) + + devList := object.VirtualDeviceList(o.Config.Hardware.Device) + l := devList.SelectByBackingInfo(&types.VirtualDiskFlatVer2BackingInfo{ + VirtualDeviceFileBackingInfo: types.VirtualDeviceFileBackingInfo{ + FileName: fmt.Sprintf("[%s] %s/disk-0.vmdk", dso.Name, vcVM.Name()), + }, + }) + ExpectWithOffset(1, l).To(HaveLen(1)) + + disk := l[0].(*types.VirtualDisk) + backing := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo) + + return disk, backing +} + +//nolint:unparam +func getDVPG( + ctx *builder.TestContextForVCSim, + path string) (object.NetworkReference, *object.DistributedVirtualPortgroup) { + + network, err := ctx.Finder.Network(ctx, path) + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + dvpg, ok := network.(*object.DistributedVirtualPortgroup) + ExpectWithOffset(1, ok).To(BeTrue()) + + return network, dvpg +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go new file mode 100644 index 000000000..5506214e8 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go @@ -0,0 +1,335 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere + +import ( + "encoding/json" + "fmt" + + "github.com/google/uuid" + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" +) + +// TODO: This mostly just a placeholder until we spend time on something better. Individual types +// don't make much sense since we don't lump everything under a single prereq condition anymore. +func errToConditionReasonAndMessage(err error) (string, string) { + switch { + case apierrors.IsNotFound(err): + return "NotFound", err.Error() + case apierrors.IsForbidden(err): + return "Forbidden", err.Error() + case apierrors.IsInvalid(err): + return "Invalid", err.Error() + case apierrors.IsInternalError(err): + return "InternalError", err.Error() + default: + return "GetError", err.Error() + } +} + +func GetVirtualMachineClass( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (*vmopv1.VirtualMachineClass, error) { + + key := ctrlclient.ObjectKey{Name: vmCtx.VM.Spec.ClassName, Namespace: vmCtx.VM.Namespace} + vmClass := &vmopv1.VirtualMachineClass{} + if err := k8sClient.Get(vmCtx, key, vmClass); err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionClassReady, reason, msg) + return nil, err + } + + if !vmClass.Status.Ready { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionClassReady, + "NotReady", "VirtualMachineClass is not marked as Ready") + return nil, fmt.Errorf("VirtualMachineClass is not Ready") + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionClassReady) + + return vmClass, nil +} + +func GetVirtualMachineImageSpecAndStatus( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (ctrlclient.Object, *vmopv1.VirtualMachineImageSpec, *vmopv1.VirtualMachineImageStatus, error) { + + var obj ctrlclient.Object + var spec *vmopv1.VirtualMachineImageSpec + var status *vmopv1.VirtualMachineImageStatus + + key := ctrlclient.ObjectKey{Name: vmCtx.VM.Spec.ImageName, Namespace: vmCtx.VM.Namespace} + vmImage := &vmopv1.VirtualMachineImage{} + if err := k8sClient.Get(vmCtx, key, vmImage); err != nil { + clusterVMImage := &vmopv1.ClusterVirtualMachineImage{} + + if apierrors.IsNotFound(err) { + key.Namespace = "" + err = k8sClient.Get(vmCtx, key, clusterVMImage) + } + + if err != nil { + // Don't use the k8s error as-is as we don't know to prefer the NS or cluster scoped error message. + // This is the same error/message that the prior code used. + reason, msg := "NotFound", fmt.Sprintf("Failed to get the VM's image: %s", key.Name) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady, reason, msg) + return nil, nil, nil, fmt.Errorf("%s: %w", msg, err) + } + + obj, spec, status = clusterVMImage, &clusterVMImage.Spec, &clusterVMImage.Status + } else { + obj, spec, status = vmImage, &vmImage.Spec, &vmImage.Status + } + + // TODO: Fix the image conditions so it just has a single Ready instead of bleeding the CL stuff. + if !conditions.IsTrueFromConditions(status.Conditions, vmopv1.VirtualMachineImageSyncedCondition) { + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady, + "NotReady", "VirtualMachineImage is not ready") + return nil, nil, nil, fmt.Errorf("VirtualMachineImage is not ready") + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady) + + return obj, spec, status, nil +} + +func getSecretData( + vmCtx context.VirtualMachineContextA2, + name string, + cmFallback bool, + k8sClient ctrlclient.Client) (map[string]string, error) { + + var data map[string]string + + key := ctrlclient.ObjectKey{Name: name, Namespace: vmCtx.VM.Namespace} + secret := &corev1.Secret{} + if err := k8sClient.Get(vmCtx, key, secret); err != nil { + configMap := &corev1.ConfigMap{} + + // For backwards compat if we cannot find the Secret, fallback to a ConfigMap. In v1a1, either a + // Secret and ConfigMap was supported for metadata (bootstrap) as separate fields, but v1a2 only + // supports Secrets. + if cmFallback && apierrors.IsNotFound(err) { + err = k8sClient.Get(vmCtx, key, configMap) + } + + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, err + } + + data = configMap.Data + } else { + data = make(map[string]string, len(secret.Data)) + + for k, v := range secret.Data { + data[k] = string(v) + } + } + + return data, nil +} + +func GetVirtualMachineBootstrap( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (map[string]string, map[string]string, map[string]map[string]string, error) { + + bootstrapSpec := &vmCtx.VM.Spec.Bootstrap + var secretName string + var data, vAppData map[string]string + var vAppExData map[string]map[string]string + + if cloudInit := bootstrapSpec.CloudInit; cloudInit != nil { + secretName = cloudInit.RawCloudConfig.Name + } else if sysPrep := bootstrapSpec.Sysprep; sysPrep != nil { + secretName = sysPrep.RawSysprep.Name + } + + if secretName != "" { + var err error + + data, err = getSecretData(vmCtx, secretName, true, k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, nil, nil, err + } + } + + // vApp bootstrap can be used alongside LinuxPrep/Sysprep. + if vApp := bootstrapSpec.VAppConfig; vApp != nil { + + if vApp.RawProperties != "" { + var err error + + vAppData, err = getSecretData(vmCtx, vApp.RawProperties, true, k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, nil, nil, err + } + + } else { + for _, p := range vApp.Properties { + from := p.Value.From + if from == nil { + continue + } + + if _, ok := vAppExData[from.Name]; !ok { + // Do the easy thing here and carry along each Secret's entire data. We could instead + // shoehorn this in the vAppData with a concat key using an invalid k8s name delimiter. + // TODO: Check that key exists, and/or deal with from.Optional. Too many options. + fromData, err := getSecretData(vmCtx, from.Name, false, k8sClient) + if err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady, reason, msg) + return nil, nil, nil, err + } + + if vAppExData == nil { + vAppExData = make(map[string]map[string]string) + } + vAppExData[from.Name] = fromData + } + } + } + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady) + + return data, vAppData, vAppExData, nil +} + +func GetVMSetResourcePolicy( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (*vmopv1.VirtualMachineSetResourcePolicy, error) { + + rpName := vmCtx.VM.Spec.Reserved.ResourcePolicyName + if rpName == "" { + conditions.Delete(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady) + return nil, nil + } + + key := ctrlclient.ObjectKey{Name: rpName, Namespace: vmCtx.VM.Namespace} + resourcePolicy := &vmopv1.VirtualMachineSetResourcePolicy{} + if err := k8sClient.Get(vmCtx, key, resourcePolicy); err != nil { + reason, msg := errToConditionReasonAndMessage(err) + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady, reason, msg) + return nil, err + } + + // The VirtualMachineSetResourcePolicy doesn't have a Ready condition or field but don't + // allow a VM to use a policy that's being deleted. + if !resourcePolicy.DeletionTimestamp.IsZero() { + err := fmt.Errorf("VirtualMachineSetResourcePolicy is being deleted") + conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady, + "NotReady", err.Error()) + return nil, err + } + + conditions.MarkTrue(vmCtx.VM, vmopv1.VirtualMachineConditionVMSetResourcePolicyReady) + + return resourcePolicy, nil +} + +// AddInstanceStorageVolumes checks if VM class is configured with instance storage volumes and appends the +// volumes to the VM's Spec if not already done. Return true if the VM had or now has instance storage volumes. +func AddInstanceStorageVolumes( + vmCtx context.VirtualMachineContextA2, + vmClass *vmopv1.VirtualMachineClass) bool { + + if instancestorage.IsPresent(vmCtx.VM) { + // Instance storage disks are copied from the class to the VM only once, regardless + // if the class changes. + return true + } + + is := vmClass.Spec.Hardware.InstanceStorage + if len(is.Volumes) == 0 { + return false + } + + volumes := make([]vmopv1.VirtualMachineVolume, 0, len(is.Volumes)) + + for _, isv := range is.Volumes { + name := constants.InstanceStoragePVCNamePrefix + uuid.NewString() + + vmv := vmopv1.VirtualMachineVolume{ + Name: name, + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + ReadOnly: false, + }, + InstanceVolumeClaim: &vmopv1.InstanceVolumeClaimVolumeSource{ + StorageClass: is.StorageClass, + Size: isv.Size, + }, + }, + }, + } + volumes = append(volumes, vmv) + } + + vmCtx.VM.Spec.Volumes = append(vmCtx.VM.Spec.Volumes, volumes...) + return true +} + +func GetVMClassConfigSpec(raw json.RawMessage) (*types.VirtualMachineConfigSpec, error) { + classConfigSpec, err := util.UnmarshalConfigSpecFromJSON(raw) + if err != nil { + return nil, err + } + util.SanitizeVMClassConfigSpec(classConfigSpec) + + return classConfigSpec, nil +} + +// HasPVC returns true if the VirtualMachine spec has a Persistent Volume claim. +func HasPVC(vmSpec vmopv1.VirtualMachineSpec) bool { + for _, vol := range vmSpec.Volumes { + if vol.PersistentVolumeClaim != nil { + return true + } + } + return false +} + +// HardwareVersionForPVCandPCIDevices returns a hardware version for VMs with PVCs and PCI devices(vGPUs/DDPIO devices) +// The hardware version is determined based on the below criteria: VMs with +// - Persistent Volume Claim (PVC) get the max(the image hardware version, minimum supported virtual hardware version for persistent volumes) +// - vGPUs/DDPIO devices get the max(the image hardware version, minimum supported virtual hardware version for PCI devices) +// - Both vGPU/DDPIO devices and PVCs get the max(the image hardware version, minimum supported virtual hardware version for PCI devices) +// - none of the above returns 0. +func HardwareVersionForPVCandPCIDevices(imageHWVersion int32, configSpec *types.VirtualMachineConfigSpec, hasPVC bool) int32 { + var configSpecHWVersion int32 + configSpecDevs := util.DevicesFromConfigSpec(configSpec) + + if len(util.SelectNvidiaVgpu(configSpecDevs)) > 0 || len(util.SelectDynamicDirectPathIO(configSpecDevs)) > 0 { + configSpecHWVersion = constants.MinSupportedHWVersionForPCIPassthruDevices + if imageHWVersion != 0 && imageHWVersion > constants.MinSupportedHWVersionForPCIPassthruDevices { + configSpecHWVersion = imageHWVersion + } + } else if hasPVC { + configSpecHWVersion = constants.MinSupportedHWVersionForPVC + if imageHWVersion != 0 && imageHWVersion > constants.MinSupportedHWVersionForPVC { + configSpecHWVersion = imageHWVersion + } + } + + return configSpecHWVersion +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go new file mode 100644 index 000000000..cf7fe642d --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go @@ -0,0 +1,629 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + goctx "context" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware/govmomi/vim25/types" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" + vsphere "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/instancestorage" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func vmUtilTests() { + + var ( + k8sClient client.Client + initObjects []client.Object + + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + vm := builder.DummyBasicVirtualMachineA2("test-vm", "dummy-ns") + + vmCtx = context.VirtualMachineContextA2{ + Context: goctx.WithValue(goctx.Background(), context.MaxDeployThreadsContextKey, 16), + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + }) + + JustBeforeEach(func() { + k8sClient = builder.NewFakeClient(initObjects...) + }) + + AfterEach(func() { + k8sClient = nil + initObjects = nil + }) + + Context("GetVirtualMachineClass", func() { + oldNamespacedVMClassFSSEnabledFunc := lib.IsNamespacedVMClassFSSEnabled + + // NOTE: As we currently have it, v1a2 must have this enabled. + When("WCP_Namespaced_VM_Class FSS is enabled", func() { + var ( + vmClass *vmopv1.VirtualMachineClass + ) + + BeforeEach(func() { + vmClass = builder.DummyVirtualMachineClass2A2("dummy-vm-class") + vmClass.Namespace = vmCtx.VM.Namespace + vmCtx.VM.Spec.ClassName = vmClass.Name + + lib.IsNamespacedVMClassFSSEnabled = func() bool { + return true + } + }) + + AfterEach(func() { + lib.IsNamespacedVMClassFSSEnabled = oldNamespacedVMClassFSSEnabledFunc + }) + + Context("VirtualMachineClass custom resource doesn't exist", func() { + It("Returns error and sets condition when VM Class does not exist", func() { + expectedErrMsg := fmt.Sprintf("virtualmachineclasses.vmoperator.vmware.com %q not found", vmCtx.VM.Spec.ClassName) + + _, err := vsphere.GetVirtualMachineClass(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(expectedErrMsg)) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.VirtualMachineConditionClassReady, "NotFound", expectedErrMsg), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + Context("VirtualMachineClass custom resource exists", func() { + + When("Is not Ready", func() { + + BeforeEach(func() { + vmClass.Status.Ready = false + initObjects = append(initObjects, vmClass) + }) + + It("returns an error", func() { + _, err := vsphere.GetVirtualMachineClass(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("VirtualMachineClass is not Ready")) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition( + vmopv1.VirtualMachineConditionClassReady, + "NotReady", + "VirtualMachineClass is not marked as Ready"), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + When("Is Ready", func() { + + BeforeEach(func() { + vmClass.Status.Ready = true + initObjects = append(initObjects, vmClass) + }) + + It("returns success", func() { + class, err := vsphere.GetVirtualMachineClass(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(class).ToNot(BeNil()) + }) + }) + }) + }) + }) + + Context("GetVMImageStatusAndContentLibraryUUID", func() { + + // NOTE: As we currently have it, v1a2 must have this enabled. + When("WCPVMImageRegistry FSS is enabled", func() { + + var ( + nsVMImage *vmopv1.VirtualMachineImage + clusterVMImage *vmopv1.ClusterVirtualMachineImage + ) + + BeforeEach(func() { + nsVMImage = builder.DummyVirtualMachineImageA2("dummy-ns-vm-image") + nsVMImage.Namespace = vmCtx.VM.Namespace + conditions.MarkTrue(nsVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + clusterVMImage = builder.DummyClusterVirtualMachineImageA2("dummy-cluster-vm-image") + conditions.MarkTrue(clusterVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + + lib.IsWCPVMImageRegistryEnabled = func() bool { + return true + } + }) + + When("Neither cluster or namespace scoped VM image exists", func() { + + It("returns error and sets condition", func() { + _, _, _, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + expectedErrMsg := fmt.Sprintf("Failed to get the VM's image: %s", vmCtx.VM.Spec.ImageName) + Expect(err.Error()).To(ContainSubstring(expectedErrMsg)) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition(vmopv1.VirtualMachineConditionImageReady, "NotFound", expectedErrMsg), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + When("VM image exists but the image is not ready", func() { + + BeforeEach(func() { + conditions.MarkFalse(nsVMImage, vmopv1.VirtualMachineImageSyncedCondition, "NotReady", "") // XXX Until rollup condition + initObjects = append(initObjects, nsVMImage) + vmCtx.VM.Spec.ImageName = nsVMImage.Name + }) + + It("returns error and sets VM condition", func() { + _, _, _, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + expectedErrMsg := "VirtualMachineImage is not ready" + Expect(err.Error()).To(ContainSubstring(expectedErrMsg)) + + expectedCondition := []metav1.Condition{ + *conditions.FalseCondition( + vmopv1.VirtualMachineConditionImageReady, "NotReady", expectedErrMsg), + } + Expect(vmCtx.VM.Status.Conditions).To(conditions.MatchConditions(expectedCondition)) + }) + }) + + When("Namespace scoped VirtualMachineImage exists and ready", func() { + BeforeEach(func() { + conditions.MarkTrue(nsVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + initObjects = append(initObjects, nsVMImage) + vmCtx.VM.Spec.ImageName = nsVMImage.Name + }) + + It("returns success", func() { + imgObj, spec, status, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(imgObj).ToNot(BeNil()) + Expect(imgObj.GetObjectKind().GroupVersionKind().Kind).To(Equal("VirtualMachineImage")) + Expect(spec).ToNot(BeNil()) + Expect(status).ToNot(BeNil()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + }) + }) + + When("ClusterVirtualMachineImage exists and ready", func() { + BeforeEach(func() { + conditions.MarkTrue(clusterVMImage, vmopv1.VirtualMachineImageSyncedCondition) // XXX Until rollup condition + initObjects = append(initObjects, clusterVMImage) + vmCtx.VM.Spec.ImageName = clusterVMImage.Name + }) + + It("returns success", func() { + imgObj, spec, status, err := vsphere.GetVirtualMachineImageSpecAndStatus(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(imgObj).ToNot(BeNil()) + Expect(imgObj.GetObjectKind().GroupVersionKind().Kind).To(Equal("ClusterVirtualMachineImage")) + Expect(spec).ToNot(BeNil()) + Expect(status).ToNot(BeNil()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionImageReady)).To(BeTrue()) + }) + }) + }) + }) + + Context("GetVirtualMachineBootstrap", func() { + const dataName = "dummy-vm-bootstrap-data" + const vAppDataName = "dummy-vm-bootstrap-vapp-data" + + var ( + bootstrapCM *corev1.ConfigMap + bootstrapSecret *corev1.Secret + bootstrapVAppCM *corev1.ConfigMap + ) + + BeforeEach(func() { + bootstrapCM = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: dataName, + Namespace: vmCtx.VM.Namespace, + }, + Data: map[string]string{ + "foo": "bar", + }, + } + + bootstrapSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: dataName, + Namespace: vmCtx.VM.Namespace, + }, + Data: map[string][]byte{ + "foo1": []byte("bar1"), + }, + } + + bootstrapVAppCM = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: vAppDataName, + Namespace: vmCtx.VM.Namespace, + }, + Data: map[string]string{ + "foo-vapp": "bar-vapp", + }, + } + }) + + When("Bootstrap via CloudInit", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap = vmopv1.VirtualMachineBootstrapSpec{ + CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{}, + } + vmCtx.VM.Spec.Bootstrap.CloudInit.RawCloudConfig.Name = dataName + }) + + It("return an error when resources does not exist", func() { + _, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeFalse()) + }) + + When("ConfigMap exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM) + }) + + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo", "bar")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + + When("Secret exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM, bootstrapSecret) + }) + + When("Prefers Secret over ConfigMap", func() { + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + // Prefer Secret over ConfigMap. + Expect(data).To(HaveKeyWithValue("foo1", "bar1")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + }) + }) + + When("Bootstrap via Sysprep", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap = vmopv1.VirtualMachineBootstrapSpec{ + Sysprep: &vmopv1.VirtualMachineBootstrapSysprepSpec{}, + } + vmCtx.VM.Spec.Bootstrap.Sysprep.RawSysprep.Name = dataName + }) + + It("return an error when resource does not exist", func() { + _, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeFalse()) + }) + + When("ConfigMap exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM) + }) + + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo", "bar")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + + When("Secret exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapCM, bootstrapSecret) + }) + + When("Prefers Secret over ConfigMap", func() { + It("returns success", func() { + data, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo1", "bar1")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + }) + }) + + When("Bootstrap with vAppConfig", func() { + + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap = vmopv1.VirtualMachineBootstrapSpec{ + VAppConfig: &vmopv1.VirtualMachineBootstrapVAppConfigSpec{}, + } + vmCtx.VM.Spec.Bootstrap.VAppConfig.RawProperties = vAppDataName + }) + + It("return an error when resource does not exist", func() { + _, _, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeFalse()) + }) + + When("ConfigMap exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, bootstrapVAppCM) + }) + + It("returns success", func() { + _, data, _, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveKeyWithValue("foo-vapp", "bar-vapp")) + Expect(conditions.IsTrue(vmCtx.VM, vmopv1.VirtualMachineConditionBootstrapReady)).To(BeTrue()) + }) + }) + + When("vAppConfig with properties", func() { + BeforeEach(func() { + vmCtx.VM.Spec.Bootstrap.VAppConfig.Properties = []common.KeyValueOrSecretKeySelectorPair{ + { + Value: common.ValueOrSecretKeySelector{ + From: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: vAppDataName, + }, + Key: "foo-vapp", + }, + }, + }, + } + + It("returns success", func() { + _, _, exData, err := vsphere.GetVirtualMachineBootstrap(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(exData).To(HaveKey(vAppDataName)) + data := exData[vAppDataName] + Expect(data).To(HaveKeyWithValue("foo-vapp", "bar-vapp")) + }) + }) + }) + }) + }) + + Context("GetVMSetResourcePolicy", func() { + + var ( + vmResourcePolicy *vmopv1.VirtualMachineSetResourcePolicy + ) + + BeforeEach(func() { + vmResourcePolicy = &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dummy-vm-rp", + Namespace: vmCtx.VM.Namespace, + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{Name: "fooRP"}, + Folder: "fooFolder", + }, + } + }) + + It("returns success when VM does not have SetResourcePolicy", func() { + vmCtx.VM.Spec.Reserved.ResourcePolicyName = "" + rp, err := vsphere.GetVMSetResourcePolicy(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(rp).To(BeNil()) + }) + + It("VM SetResourcePolicy does not exist", func() { + vmCtx.VM.Spec.Reserved.ResourcePolicyName = "bogus" + rp, err := vsphere.GetVMSetResourcePolicy(vmCtx, k8sClient) + Expect(err).To(HaveOccurred()) + Expect(rp).To(BeNil()) + }) + + When("VM SetResourcePolicy exists", func() { + BeforeEach(func() { + initObjects = append(initObjects, vmResourcePolicy) + vmCtx.VM.Spec.Reserved.ResourcePolicyName = vmResourcePolicy.Name + }) + + It("returns success", func() { + rp, err := vsphere.GetVMSetResourcePolicy(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(rp).ToNot(BeNil()) + }) + }) + }) + + Context("AddInstanceStorageVolumes", func() { + + var ( + vmClass *vmopv1.VirtualMachineClass + ) + + expectInstanceStorageVolumes := func( + vm *vmopv1.VirtualMachine, + isStorage vmopv1.InstanceStorage) { + + ExpectWithOffset(1, isStorage.Volumes).ToNot(BeEmpty()) + isVolumes := instancestorage.FilterVolumes(vm) + ExpectWithOffset(1, isVolumes).To(HaveLen(len(isStorage.Volumes))) + + for _, isVol := range isStorage.Volumes { + found := false + + for idx, vol := range isVolumes { + claim := vol.PersistentVolumeClaim.InstanceVolumeClaim + if claim.StorageClass == isStorage.StorageClass && claim.Size == isVol.Size { + isVolumes = append(isVolumes[:idx], isVolumes[idx+1:]...) + found = true + break + } + } + + ExpectWithOffset(1, found).To(BeTrue(), "failed to find instance storage volume for %v", isVol) + } + } + + BeforeEach(func() { + vmClass = builder.DummyVirtualMachineClassA2() + }) + + When("InstanceStorage FFS is enabled", func() { + + It("VM Class does not contain instance storage volumes", func() { + is := vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeFalse()) + Expect(instancestorage.FilterVolumes(vmCtx.VM)).To(BeEmpty()) + }) + + When("Instance Volume is added in VM Class", func() { + BeforeEach(func() { + vmClass.Spec.Hardware.InstanceStorage = builder.DummyInstanceStorageA2() + }) + + It("Instance Volumes should be added", func() { + is := vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeTrue()) + expectInstanceStorageVolumes(vmCtx.VM, vmClass.Spec.Hardware.InstanceStorage) + }) + + It("Instance Storage is already added to VM Spec.Volumes", func() { + is := vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeTrue()) + + isVolumesBefore := instancestorage.FilterVolumes(vmCtx.VM) + expectInstanceStorageVolumes(vmCtx.VM, vmClass.Spec.Hardware.InstanceStorage) + + // Instance Storage is already configured, should not patch again + is = vsphere.AddInstanceStorageVolumes(vmCtx, vmClass) + Expect(is).To(BeTrue()) + isVolumesAfter := instancestorage.FilterVolumes(vmCtx.VM) + Expect(isVolumesAfter).To(HaveLen(len(isVolumesBefore))) + Expect(isVolumesAfter).To(Equal(isVolumesBefore)) + }) + }) + }) + }) + + Context("HasPVC", func() { + + Context("Spec has no PVC", func() { + It("will return false", func() { + spec := vmopv1.VirtualMachineSpec{} + Expect(vsphere.HasPVC(spec)).To(BeFalse()) + }) + }) + + Context("Spec has PVCs", func() { + It("will return true", func() { + spec := vmopv1.VirtualMachineSpec{ + Volumes: []vmopv1.VirtualMachineVolume{ + { + Name: "dummy-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-claim-1", + }, + }, + }, + }, + }, + } + Expect(vsphere.HasPVC(spec)).To(BeTrue()) + }) + }) + }) + + Context("HardwareVersionForPVCandPCIDevices", func() { + var ( + configSpec *types.VirtualMachineConfigSpec + imageHWVersion int32 + ) + + BeforeEach(func() { + imageHWVersion = 14 + configSpec = &types.VirtualMachineConfigSpec{ + Name: "dummy-VM", + DeviceChange: []types.BaseVirtualDeviceConfigSpec{ + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughVmiopBackingInfo{ + Vgpu: "profile-from-configspec", + }, + }, + }, + }, + &types.VirtualDeviceConfigSpec{ + Operation: types.VirtualDeviceConfigSpecOperationAdd, + Device: &types.VirtualPCIPassthrough{ + VirtualDevice: types.VirtualDevice{ + Backing: &types.VirtualPCIPassthroughDynamicBackingInfo{ + AllowedDevice: []types.VirtualPCIPassthroughAllowedDevice{ + { + VendorId: 52, + DeviceId: 53, + }, + }, + CustomLabel: "label-from-configspec", + }, + }, + }, + }, + }, + } + }) + + It("ConfigSpec has PCI devices and VM spec has PVCs", func() { + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(17))) + }) + + It("ConfigSpec has PCI devices and VM spec has no PVCs", func() { + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, false)).To(Equal(int32(17))) + }) + + It("ConfigSpec has PCI devices, VM spec has PVCs image hardware version is higher than min supported HW version for PCI devices", func() { + imageHWVersion = 18 + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(18))) + }) + + It("VM spec has PVCs and config spec has no devices", func() { + configSpec = &types.VirtualMachineConfigSpec{} + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(15))) + }) + + It("VM spec has PVCs, config spec has no devices and image hardware version is higher than min supported PVC HW version", func() { + configSpec = &types.VirtualMachineConfigSpec{} + imageHWVersion = 16 + Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(16))) + }) + }) +} diff --git a/pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go b/pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go new file mode 100644 index 000000000..ac42dce8a --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/vsphere_suite_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2021 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package vsphere_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +var suite = builder.NewTestSuite() + +func vcSimTests() { + Describe("CPUFreq", cpuFreqTests) + Describe("InitOvfCacheAndLockPool", initOvfCacheAndLockPoolTests) + Describe("ResourcePolicyTests", resourcePolicyTests) + Describe("VirtualMachine", vmTests) + Describe("VirtualMachineE2E", vmE2ETests) + Describe("VirtualMachineUtilsTest", vmUtilTests) +} + +func TestVSphereProvider(t *testing.T) { + suite.Register(t, "VMProvider Tests", nil, vcSimTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/test/builder/fake.go b/test/builder/fake.go index 7aab65f77..5764ceac1 100644 --- a/test/builder/fake.go +++ b/test/builder/fake.go @@ -42,13 +42,15 @@ func KnownObjectTypes() []client.Object { &v1alpha2.VirtualMachineService{}, &v1alpha1.VirtualMachineClass{}, &v1alpha2.VirtualMachineClass{}, - &cnsv1alpha1.CnsNodeVmAttachment{}, &v1alpha1.VirtualMachinePublishRequest{}, &v1alpha2.VirtualMachinePublishRequest{}, &v1alpha1.ClusterVirtualMachineImage{}, &v1alpha2.ClusterVirtualMachineImage{}, &v1alpha1.VirtualMachineImage{}, &v1alpha2.VirtualMachineImage{}, + &cnsv1alpha1.CnsNodeVmAttachment{}, + &ncpv1alpha1.VirtualNetworkInterface{}, + &netopv1alpha1.NetworkInterface{}, } } diff --git a/test/builder/utila2.go b/test/builder/utila2.go index a36c8bc11..902587367 100644 --- a/test/builder/utila2.go +++ b/test/builder/utila2.go @@ -14,100 +14,26 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" ) -func DummyVirtualMachineSetResourcePolicyA2() *vmopv1.VirtualMachineSetResourcePolicy { - return &vmopv1.VirtualMachineSetResourcePolicy{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-", - }, - Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ - ResourcePool: vmopv1.ResourcePoolSpec{ - Name: "dummy-resource-pool", - Reservations: vmopv1.VirtualMachineResourceSpec{ - Cpu: resource.MustParse("1Gi"), - Memory: resource.MustParse("2Gi"), - }, - Limits: vmopv1.VirtualMachineResourceSpec{ - Cpu: resource.MustParse("2Gi"), - Memory: resource.MustParse("4Gi"), - }, - }, - Folder: "dummy-folder", - ClusterModuleGroups: []string{"dummy-cluster-modules"}, - }, - } -} - -func DummyVirtualMachineServiceA2() *vmopv1.VirtualMachineService { - return &vmopv1.VirtualMachineService{ +func DummyVirtualMachineClass2A2(name string) *vmopv1.VirtualMachineClass { + return &vmopv1.VirtualMachineClass{ ObjectMeta: metav1.ObjectMeta{ - // Using image.GenerateName causes problems with unit tests - Name: fmt.Sprintf("test-%s", uuid.New()), + Name: name, }, - Spec: vmopv1.VirtualMachineServiceSpec{ - Type: vmopv1.VirtualMachineServiceTypeLoadBalancer, - Ports: []vmopv1.VirtualMachineServicePort{ - { - Name: "dummy-port", - Protocol: "TCP", - Port: 42, - TargetPort: 4242, - }, - }, - Selector: map[string]string{ - "foo": "bar", + Spec: vmopv1.VirtualMachineClassSpec{ + Hardware: vmopv1.VirtualMachineClassHardware{ + Cpus: int64(2), + Memory: resource.MustParse("4Gi"), }, - }, - } -} - -func DummyVirtualMachineA2() *vmopv1.VirtualMachine { - return &vmopv1.VirtualMachine{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-", - Labels: map[string]string{}, - Annotations: map[string]string{}, - }, - Spec: vmopv1.VirtualMachineSpec{ - ImageName: DummyImageName, - ClassName: DummyClassName, - PowerState: vmopv1.VirtualMachinePowerStateOn, - Volumes: []vmopv1.VirtualMachineVolume{ - { - Name: DummyVolumeName, - VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ - PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ - PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: DummyPVCName, - }, - }, + Policies: vmopv1.VirtualMachineClassPolicies{ + Resources: vmopv1.VirtualMachineClassResources{ + Requests: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("1Gi"), + Memory: resource.MustParse("2Gi"), + }, + Limits: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("2Gi"), + Memory: resource.MustParse("4Gi"), }, - }, - }, - }, - } -} - -func DummyVirtualMachinePublishRequestA2(name, namespace, sourceName, itemName, clName string) *vmopv1.VirtualMachinePublishRequest { - return &vmopv1.VirtualMachinePublishRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Finalizers: []string{"virtualmachinepublishrequest.vmoperator.vmware.com"}, - }, - Spec: vmopv1.VirtualMachinePublishRequestSpec{ - Source: vmopv1.VirtualMachinePublishRequestSource{ - Name: sourceName, - APIVersion: "vmoperator.vmware.com/v1alpha2", - Kind: "VirtualMachine", - }, - Target: vmopv1.VirtualMachinePublishRequestTarget{ - Item: vmopv1.VirtualMachinePublishRequestTargetItem{ - Name: itemName, - }, - Location: vmopv1.VirtualMachinePublishRequestTargetLocation{ - Name: clName, - APIVersion: "imageregistry.vmware.com/v1alpha1", - Kind: "ContentLibrary", }, }, }, @@ -140,6 +66,20 @@ func DummyVirtualMachineClassA2() *vmopv1.VirtualMachineClass { } } +func DummyInstanceStorageA2() vmopv1.InstanceStorage { + return vmopv1.InstanceStorage{ + StorageClass: DummyStorageClassName, + Volumes: []vmopv1.InstanceStorageVolume{ + { + Size: resource.MustParse("256Gi"), + }, + { + Size: resource.MustParse("512Gi"), + }, + }, + } +} + func DummyInstanceStorageVirtualMachineVolumesA2() []vmopv1.VirtualMachineVolume { return []vmopv1.VirtualMachineVolume{ { @@ -173,6 +113,161 @@ func DummyInstanceStorageVirtualMachineVolumesA2() []vmopv1.VirtualMachineVolume } } +func DummyBasicVirtualMachineA2(name, namespace string) *vmopv1.VirtualMachine { + return &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + Spec: vmopv1.VirtualMachineSpec{ + ImageName: DummyImageName, + ClassName: DummyClassName, + PowerState: vmopv1.VirtualMachinePowerStateOn, + PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, + SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + }, + } +} + +func DummyVirtualMachineA2() *vmopv1.VirtualMachine { + return &vmopv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + Spec: vmopv1.VirtualMachineSpec{ + ImageName: DummyImageName, + ClassName: DummyClassName, + PowerState: vmopv1.VirtualMachinePowerStateOn, + PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, + SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + Volumes: []vmopv1.VirtualMachineVolume{ + { + Name: DummyVolumeName, + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: DummyPVCName, + }, + }, + }, + }, + }, + /* TODO: Convert this if/as needed + NetworkInterfaces: []vmopv1.VirtualMachineNetworkInterface{ + { + NetworkName: DummyNetworkName, + NetworkType: "", + }, + { + NetworkName: DummyNetworkName + "-2", + NetworkType: "", + }, + }, + VmMetadata: &vmopv1.VirtualMachineMetadata{ + ConfigMapName: DummyMetadataCMName, + Transport: "ExtraConfig", + }, + */ + }, + } +} + +func AddDummyInstanceStorageVolumeA2(vm *vmopv1.VirtualMachine) { + vm.Spec.Volumes = append(vm.Spec.Volumes, DummyInstanceStorageVirtualMachineVolumesA2()...) +} + +func DummyVirtualMachineServiceA2() *vmopv1.VirtualMachineService { + return &vmopv1.VirtualMachineService{ + ObjectMeta: metav1.ObjectMeta{ + // Using image.GenerateName causes problems with unit tests + Name: fmt.Sprintf("test-%s", uuid.New()), + }, + Spec: vmopv1.VirtualMachineServiceSpec{ + Type: vmopv1.VirtualMachineServiceTypeLoadBalancer, + Ports: []vmopv1.VirtualMachineServicePort{ + { + Name: "dummy-port", + Protocol: "TCP", + Port: 42, + TargetPort: 4242, + }, + }, + Selector: map[string]string{ + "foo": "bar", + }, + }, + } +} + +func DummyVirtualMachineSetResourcePolicyA2() *vmopv1.VirtualMachineSetResourcePolicy { + return &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{ + Name: "dummy-resource-pool", + Reservations: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("1Gi"), + Memory: resource.MustParse("2Gi"), + }, + Limits: vmopv1.VirtualMachineResourceSpec{ + Cpu: resource.MustParse("2Gi"), + Memory: resource.MustParse("4Gi"), + }, + }, + Folder: "dummy-folder", + ClusterModuleGroups: []string{"dummy-cluster-modules"}, + }, + } +} + +func DummyVirtualMachineSetResourcePolicy2A2(name, namespace string) *vmopv1.VirtualMachineSetResourcePolicy { + return &vmopv1.VirtualMachineSetResourcePolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: vmopv1.VirtualMachineSetResourcePolicySpec{ + ResourcePool: vmopv1.ResourcePoolSpec{ + Name: name, + }, + Folder: name, + }, + } +} + +func DummyVirtualMachinePublishRequestA2(name, namespace, sourceName, itemName, clName string) *vmopv1.VirtualMachinePublishRequest { + return &vmopv1.VirtualMachinePublishRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Finalizers: []string{"virtualmachinepublishrequest.vmoperator.vmware.com"}, + }, + Spec: vmopv1.VirtualMachinePublishRequestSpec{ + Source: vmopv1.VirtualMachinePublishRequestSource{ + Name: sourceName, + APIVersion: "vmoperator.vmware.com/v1alpha2", + Kind: "VirtualMachine", + }, + Target: vmopv1.VirtualMachinePublishRequestTarget{ + Item: vmopv1.VirtualMachinePublishRequestTargetItem{ + Name: itemName, + }, + Location: vmopv1.VirtualMachinePublishRequestTargetLocation{ + Name: clName, + APIVersion: "imageregistry.vmware.com/v1alpha1", + Kind: "ContentLibrary", + }, + }, + }, + } +} + func DummyVirtualMachineImageA2(imageName string) *vmopv1.VirtualMachineImage { return &vmopv1.VirtualMachineImage{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/builder/vcsim_test_context.go b/test/builder/vcsim_test_context.go index 9805a311f..daad84acc 100644 --- a/test/builder/vcsim_test_context.go +++ b/test/builder/vcsim_test_context.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021 VMware, Inc. All Rights Reserved. +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Package builder is a comment just to silence the linter @@ -45,8 +45,11 @@ import ( _ "github.com/vmware/govmomi/vapi/cluster/simulator" _ "github.com/vmware/govmomi/vapi/simulator" - vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/conditions2" "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/record" "github.com/vmware-tanzu/vm-operator/test/testutil" @@ -55,8 +58,9 @@ import ( type NetworkEnv string const ( - NetworkEnvVDS = NetworkEnv("vds") - NetworkEnvNSXT = NetworkEnv("nsx-t") + NetworkEnvVDS = NetworkEnv("vds") + NetworkEnvNSXT = NetworkEnv("nsx-t") + NetworkEnvNamed = NetworkEnv("named") NsxTLogicalSwitchUUID = "nsxt-dummy-ls-uuid" ) @@ -96,6 +100,9 @@ type VCSimTestConfig struct { // WithVMClassAsConfigDaynDate enables the WCP_VM_CLASS_AS_CONFIG_DAYNDATE FSS. WithVMClassAsConfigDaynDate bool + // WithV1A2 enables the VMServiceV1Alpha2FSS FSS. + WithV1A2 bool + // WithNetworkEnv is the network environment type. WithNetworkEnv NetworkEnv } @@ -137,6 +144,7 @@ type TestContextForVCSim struct { folder *object.Folder datastore *object.Datastore withFaultDomains bool + withV1A2 bool singleCCR *object.ClusterComputeResource azCCRs map[string][]*object.ClusterComputeResource @@ -160,7 +168,7 @@ func (s *TestSuite) NewTestContextForVCSim( ctx := newTestContextForVCSim(config, initObjects) - ctx.setupEnvFSS(config) + ctx.setupEnv(config) ctx.setupVCSim(config) ctx.setupContentLibrary(config) ctx.setupK8sConfig(config) @@ -180,6 +188,7 @@ func newTestContextForVCSim( PodNamespace: "vmop-pod-test", Recorder: fakeRecorder, withFaultDomains: config.WithFaultDomains, + withV1A2: config.WithV1A2, } if ctx.withFaultDomains { @@ -272,19 +281,22 @@ func (c *TestContextForVCSim) CreateWorkloadNamespace() WorkloadNamespaceInfo { Expect(c.Client.Update(c, ns)).To(Succeed()) } - if clID := c.ContentLibraryID; clID != "" { - csBinding := &vmopv1.ContentSourceBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: clID, - Namespace: ns.Name, - }, - ContentSourceRef: vmopv1.ContentSourceReference{ - APIVersion: vmopv1.SchemeGroupVersion.Group, - Kind: "ContentSource", - Name: clID, - }, + // Not the exact right FFS, but it's what we've plumbed and is otherwise implied. + if !c.withV1A2 { + if clID := c.ContentLibraryID; clID != "" { + csBinding := &v1alpha1.ContentSourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: clID, + Namespace: ns.Name, + }, + ContentSourceRef: v1alpha1.ContentSourceReference{ + APIVersion: v1alpha1.SchemeGroupVersion.Group, + Kind: "ContentSource", + Name: clID, + }, + } + Expect(c.Client.Create(c, csBinding)).To(Succeed()) } - Expect(c.Client.Create(c, csBinding)).To(Succeed()) } resourceQuota := &corev1.ResourceQuota{ @@ -313,10 +325,26 @@ func (c *TestContextForVCSim) CreateWorkloadNamespace() WorkloadNamespaceInfo { } } -// TODO: Get rid of runtime env checks so this isn't needed. -func (c *TestContextForVCSim) setupEnvFSS(config VCSimTestConfig) { +func (c *TestContextForVCSim) setupEnv(config VCSimTestConfig) { Expect(lib.SetVMOpNamespaceEnv(c.PodNamespace)).To(Succeed()) + switch config.WithNetworkEnv { + case NetworkEnvVDS: + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeVDS)).To(Succeed()) + case NetworkEnvNSXT: + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeNSXT)).To(Succeed()) + case NetworkEnvNamed: + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeNamed)).To(Succeed()) + default: + Expect(os.Unsetenv(lib.NetworkProviderType)).To(Succeed()) + } + + v1a2 := "false" + if config.WithV1A2 { + v1a2 = "true" + } + Expect(os.Setenv(lib.VMServiceV1Alpha2FSS, v1a2)).To(Succeed()) + if config.WithContentLibrary { Expect(os.Setenv("CONTENT_API_WAIT_SECS", "1")).To(Succeed()) } @@ -465,32 +493,6 @@ func (c *TestContextForVCSim) setupContentLibrary(config VCSimTestConfig) { Expect(clID).ToNot(BeEmpty()) c.ContentLibraryID = clID - clProvider := &vmopv1.ContentLibraryProvider{ - ObjectMeta: metav1.ObjectMeta{ - Name: clID, - }, - Spec: vmopv1.ContentLibraryProviderSpec{ - UUID: clID, - }, - } - Expect(c.Client.Create(c, clProvider)).To(Succeed()) - - cs := &vmopv1.ContentSource{ - ObjectMeta: metav1.ObjectMeta{ - Name: clID, - }, - Spec: vmopv1.ContentSourceSpec{ - ProviderRef: vmopv1.ContentProviderReference{ - Name: clProvider.Name, - Kind: "ContentLibraryProvider", - }, - }, - } - Expect(c.Client.Create(c, cs)).To(Succeed()) - - Expect(controllerutil.SetOwnerReference(cs, clProvider, c.Client.Scheme())).To(Succeed()) - Expect(c.Client.Update(c, clProvider)).To(Succeed()) - libraryItem := library.Item{ Name: "test-image-ovf", Type: "ovf", @@ -498,12 +500,50 @@ func (c *TestContextForVCSim) setupContentLibrary(config VCSimTestConfig) { } c.ContentLibraryImageName = libraryItem.Name - vmImage := DummyVirtualMachineImage(c.ContentLibraryImageName) - Expect(controllerutil.SetOwnerReference(clProvider, vmImage, c.Client.Scheme())).To(Succeed()) - Expect(c.Client.Create(c, vmImage)).To(Succeed()) - - createContentLibraryItem(libMgr, libraryItem, + itemID := createContentLibraryItem(libMgr, libraryItem, path.Join(testutil.GetRootDirOrDie(), "images", "ttylinux-pc_i486-16.1.ovf")) + + // Not the exact right FFS, but it's what we've plumbed and is otherwise implied. + if c.withV1A2 { + // The image isn't quite as prod but sufficient for what we need here ATM. + clusterVMImage := DummyClusterVirtualMachineImageA2(c.ContentLibraryImageName) + clusterVMImage.Spec.ProviderRef.Kind = "ClusterContentLibraryItem" + Expect(c.Client.Create(c, clusterVMImage)).To(Succeed()) + clusterVMImage.Status.ProviderItemID = itemID + conditions2.MarkTrue(clusterVMImage, v1alpha2.VirtualMachineImageSyncedCondition) // TODO: Until we get rollup Ready condition + Expect(c.Client.Status().Update(c, clusterVMImage)).To(Succeed()) + + } else { + clProvider := &v1alpha1.ContentLibraryProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: clID, + }, + Spec: v1alpha1.ContentLibraryProviderSpec{ + UUID: clID, + }, + } + Expect(c.Client.Create(c, clProvider)).To(Succeed()) + + cs := &v1alpha1.ContentSource{ + ObjectMeta: metav1.ObjectMeta{ + Name: clID, + }, + Spec: v1alpha1.ContentSourceSpec{ + ProviderRef: v1alpha1.ContentProviderReference{ + Name: clProvider.Name, + Kind: "ContentLibraryProvider", + }, + }, + } + Expect(c.Client.Create(c, cs)).To(Succeed()) + + Expect(controllerutil.SetOwnerReference(cs, clProvider, c.Client.Scheme())).To(Succeed()) + Expect(c.Client.Update(c, clProvider)).To(Succeed()) + + vmImage := DummyVirtualMachineImage(c.ContentLibraryImageName) + Expect(controllerutil.SetOwnerReference(clProvider, vmImage, c.Client.Scheme())).To(Succeed()) + Expect(c.Client.Create(c, vmImage)).To(Succeed()) + } } func (c *TestContextForVCSim) ContentLibraryItemTemplate(srcVMName, templateName string) { @@ -529,21 +569,30 @@ func (c *TestContextForVCSim) ContentLibraryItemTemplate(srcVMName, templateName }, } - _, err = vcenter.NewManager(c.RestClient).CreateTemplate(c, spec) + itemID, err := vcenter.NewManager(c.RestClient).CreateTemplate(c, spec) Expect(err).ToNot(HaveOccurred()) // Create the expected VirtualMachineImage for the template. - vmImage := DummyVirtualMachineImage(templateName) - cl := &vmopv1.ContentLibraryProvider{} - Expect(c.Client.Get(c, client.ObjectKey{Name: clID}, cl)).To(Succeed()) - Expect(controllerutil.SetOwnerReference(cl, vmImage, c.Client.Scheme())).To(Succeed()) - Expect(c.Client.Create(c, vmImage)).To(Succeed()) + if c.withV1A2 { + clusterVMImage := DummyClusterVirtualMachineImageA2(templateName) + clusterVMImage.Spec.ProviderRef.Kind = "ClusterContentLibraryItem" + Expect(c.Client.Create(c, clusterVMImage)).To(Succeed()) + clusterVMImage.Status.ProviderItemID = itemID + conditions2.MarkTrue(clusterVMImage, v1alpha2.VirtualMachineImageSyncedCondition) // TODO: Until we get rollup Ready condition + Expect(c.Client.Status().Update(c, clusterVMImage)).To(Succeed()) + } else { + vmImage := DummyVirtualMachineImage(templateName) + cl := &v1alpha1.ContentLibraryProvider{} + Expect(c.Client.Get(c, client.ObjectKey{Name: clID}, cl)).To(Succeed()) + Expect(controllerutil.SetOwnerReference(cl, vmImage, c.Client.Scheme())).To(Succeed()) + Expect(c.Client.Create(c, vmImage)).To(Succeed()) + } } func createContentLibraryItem( libMgr *library.Manager, libraryItem library.Item, - itemPath string) { + itemPath string) string { ctx := goctx.Background() @@ -590,6 +639,8 @@ func createContentLibraryItem( } Expect(uploadFunc(itemPath)).To(Succeed()) Expect(libMgr.CompleteLibraryItemUpdateSession(ctx, sessionID)).To(Succeed()) + + return itemID } func (c *TestContextForVCSim) setupK8sConfig(config VCSimTestConfig) { @@ -720,11 +771,42 @@ func (c *TestContextForVCSim) GetAZClusterComputes(azName string) []*object.Clus func (c *TestContextForVCSim) CreateVirtualMachineSetResourcePolicy( name string, - nsInfo WorkloadNamespaceInfo) (*vmopv1.VirtualMachineSetResourcePolicy, *object.Folder) { + nsInfo WorkloadNamespaceInfo) (*v1alpha1.VirtualMachineSetResourcePolicy, *object.Folder) { + + ExpectWithOffset(1, c.withV1A2).To(BeFalse()) resourcePolicy := DummyVirtualMachineSetResourcePolicy2(name, nsInfo.Namespace) Expect(c.Client.Create(c, resourcePolicy)).To(Succeed()) + folder := c.createVirtualMachineSetResourcePolicyCommon( + resourcePolicy.Spec.ResourcePool.Name, + resourcePolicy.Spec.Folder.Name, + nsInfo) + + return resourcePolicy, folder +} + +func (c *TestContextForVCSim) CreateVirtualMachineSetResourcePolicyA2( + name string, + nsInfo WorkloadNamespaceInfo) (*v1alpha2.VirtualMachineSetResourcePolicy, *object.Folder) { + + ExpectWithOffset(1, c.withV1A2).To(BeTrue()) + + resourcePolicy := DummyVirtualMachineSetResourcePolicy2A2(name, nsInfo.Namespace) + Expect(c.Client.Create(c, resourcePolicy)).To(Succeed()) + + folder := c.createVirtualMachineSetResourcePolicyCommon( + resourcePolicy.Spec.ResourcePool.Name, + resourcePolicy.Spec.Folder, + nsInfo) + + return resourcePolicy, folder +} + +func (c *TestContextForVCSim) createVirtualMachineSetResourcePolicyCommon( + rpName, folderName string, + nsInfo WorkloadNamespaceInfo) *object.Folder { + var rps []*object.ResourcePool if c.withFaultDomains { @@ -749,14 +831,14 @@ func (c *TestContextForVCSim) CreateVirtualMachineSetResourcePolicy( nsRP, ok := objRef.(*object.ResourcePool) Expect(ok).To(BeTrue()) - _, err = nsRP.Create(c, resourcePolicy.Spec.ResourcePool.Name, types.DefaultResourceConfigSpec()) + _, err = nsRP.Create(c, rpName, types.DefaultResourceConfigSpec()) Expect(err).ToNot(HaveOccurred()) } - folder, err := nsInfo.Folder.CreateFolder(c, resourcePolicy.Spec.Folder.Name) + folder, err := nsInfo.Folder.CreateFolder(c, folderName) Expect(err).ToNot(HaveOccurred()) - return resourcePolicy, folder + return folder } func (c *TestContextForVCSim) GetVMFromMoID(moID string) *object.VirtualMachine { From d424e4f79b881ab90f81b64de99df1412a2902f9 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 6 Sep 2023 15:46:01 -0500 Subject: [PATCH 13/54] Enable the v1a2 webhooks with the FFS is enabled We'll likely need the existing v1a1 mutation webhooks for some types - definitely for the VM - but that can wait to be addressed later. However, we will not need both v1a1 and v1a2 validation webhooks since the v1a2 hooks will cover v1a1 after version conversion. Correct some package names from 9869e812 --- config/webhook/manifests.yaml | 72 +++++++++---------- webhooks/virtualmachine/webhooks.go | 13 +++- .../virtualmachineclass/v1alpha2/webhooks.go | 2 +- webhooks/virtualmachineclass/webhooks.go | 6 ++ .../v1alpha2/webhooks.go | 2 +- .../virtualmachinepublishrequest/webhooks.go | 5 ++ webhooks/virtualmachineservice/webhooks.go | 5 ++ .../webhooks.go | 5 ++ 8 files changed, 70 insertions(+), 40 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index faaceb654..8b124ee10 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -103,19 +103,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineclass + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachine failurePolicy: Fail - name: default.validating.virtualmachineclass.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachine.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha1 + - v1alpha2 operations: - CREATE - UPDATE resources: - - virtualmachineclasses + - virtualmachines sideEffects: None - admissionReviewVersions: - v1 @@ -124,9 +124,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinepublishrequest + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineclass failurePolicy: Fail - name: default.validating.virtualmachinepublishrequest.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachineclass.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -136,7 +136,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachinepublishrequests + - virtualmachineclasses sideEffects: None - admissionReviewVersions: - v1 @@ -145,19 +145,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineservice + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineclass failurePolicy: Fail - name: default.validating.virtualmachineservice.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachineclass.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha1 + - v1alpha2 operations: - CREATE - UPDATE resources: - - virtualmachineservices + - virtualmachineclasses sideEffects: None - admissionReviewVersions: - v1 @@ -166,9 +166,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinepublishrequest failurePolicy: Fail - name: default.validating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachinepublishrequest.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -178,7 +178,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachinesetresourcepolicies + - virtualmachinepublishrequests sideEffects: None - admissionReviewVersions: - v1 @@ -187,19 +187,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha1-webconsolerequest + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinepublishrequest failurePolicy: Fail - name: default.validating.webconsolerequest.v1alpha1.vmoperator.vmware.com + name: default.validating.virtualmachinepublishrequest.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha1 + - v1alpha2 operations: - CREATE - UPDATE resources: - - webconsolerequests + - virtualmachinepublishrequests sideEffects: None - admissionReviewVersions: - v1 @@ -208,19 +208,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachine + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachineservice failurePolicy: Fail - name: default.validating.virtualmachine.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachineservice.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha2 + - v1alpha1 operations: - CREATE - UPDATE resources: - - virtualmachines + - virtualmachineservices sideEffects: None - admissionReviewVersions: - v1 @@ -229,9 +229,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineclass + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineservice failurePolicy: Fail - name: default.validating.virtualmachineclass.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachineservice.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -241,7 +241,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachineclasses + - virtualmachineservices sideEffects: None - admissionReviewVersions: - v1 @@ -250,19 +250,19 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinepublishrequest + path: /default-validate-vmoperator-vmware-com-v1alpha1-virtualmachinesetresourcepolicy failurePolicy: Fail - name: default.validating.virtualmachinepublishrequest.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachinesetresourcepolicy.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha2 + - v1alpha1 operations: - CREATE - UPDATE resources: - - virtualmachinepublishrequests + - virtualmachinesetresourcepolicies sideEffects: None - admissionReviewVersions: - v1 @@ -271,9 +271,9 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachineservice + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinesetresourcepolicy failurePolicy: Fail - name: default.validating.virtualmachineservice.v1alpha2.vmoperator.vmware.com + name: default.validating.virtualmachinesetresourcepolicy.v1alpha2.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com @@ -283,7 +283,7 @@ webhooks: - CREATE - UPDATE resources: - - virtualmachineservices + - virtualmachinesetresourcepolicies sideEffects: None - admissionReviewVersions: - v1 @@ -292,17 +292,17 @@ webhooks: service: name: webhook-service namespace: system - path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinesetresourcepolicy + path: /default-validate-vmoperator-vmware-com-v1alpha1-webconsolerequest failurePolicy: Fail - name: default.validating.virtualmachinesetresourcepolicy.v1alpha2.vmoperator.vmware.com + name: default.validating.webconsolerequest.v1alpha1.vmoperator.vmware.com rules: - apiGroups: - vmoperator.vmware.com apiVersions: - - v1alpha2 + - v1alpha1 operations: - CREATE - UPDATE resources: - - virtualmachinesetresourcepolicies + - webconsolerequests sideEffects: None diff --git a/webhooks/virtualmachine/webhooks.go b/webhooks/virtualmachine/webhooks.go index 49bf76683..83370a430 100644 --- a/webhooks/virtualmachine/webhooks.go +++ b/webhooks/virtualmachine/webhooks.go @@ -9,12 +9,21 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { - if err := v1alpha1.AddToManager(ctx, mgr); err != nil { - return errors.Wrap(err, "failed to initialize v1alpha1 webhooks") + if lib.IsVMServiceV1Alpha2FSSEnabled() { + // TODO: We'll likely still need the v1a1 mutation wehbook (at least some limited version of it) + if err := v1alpha2.AddToManager(ctx, mgr); err != nil { + return errors.Wrap(err, "failed to initialize v1alpha2 webhooks") + } + } else { + if err := v1alpha1.AddToManager(ctx, mgr); err != nil { + return errors.Wrap(err, "failed to initialize v1alpha1 webhooks") + } } return nil diff --git a/webhooks/virtualmachineclass/v1alpha2/webhooks.go b/webhooks/virtualmachineclass/v1alpha2/webhooks.go index f727028ec..226641b85 100644 --- a/webhooks/virtualmachineclass/v1alpha2/webhooks.go +++ b/webhooks/virtualmachineclass/v1alpha2/webhooks.go @@ -1,7 +1,7 @@ // Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package virtualmachineclass +package v1alpha2 import ( "github.com/pkg/errors" diff --git a/webhooks/virtualmachineclass/webhooks.go b/webhooks/virtualmachineclass/webhooks.go index 9f6063a6c..4611b5248 100644 --- a/webhooks/virtualmachineclass/webhooks.go +++ b/webhooks/virtualmachineclass/webhooks.go @@ -7,9 +7,15 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineclass/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineclass/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } + return v1alpha1.AddToManager(ctx, mgr) } diff --git a/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go b/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go index 213223611..8c2581a5a 100644 --- a/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go +++ b/webhooks/virtualmachinepublishrequest/v1alpha2/webhooks.go @@ -1,7 +1,7 @@ // Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package virtualmachinepublishrequest +package v1alpha2 import ( "github.com/pkg/errors" diff --git a/webhooks/virtualmachinepublishrequest/webhooks.go b/webhooks/virtualmachinepublishrequest/webhooks.go index 5c3759fce..91a84119f 100644 --- a/webhooks/virtualmachinepublishrequest/webhooks.go +++ b/webhooks/virtualmachinepublishrequest/webhooks.go @@ -7,9 +7,14 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinepublishrequest/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinepublishrequest/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } return v1alpha1.AddToManager(ctx, mgr) } diff --git a/webhooks/virtualmachineservice/webhooks.go b/webhooks/virtualmachineservice/webhooks.go index 440b05332..c13880752 100644 --- a/webhooks/virtualmachineservice/webhooks.go +++ b/webhooks/virtualmachineservice/webhooks.go @@ -7,9 +7,14 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineservice/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachineservice/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } return v1alpha1.AddToManager(ctx, mgr) } diff --git a/webhooks/virtualmachinesetresourcepolicy/webhooks.go b/webhooks/virtualmachinesetresourcepolicy/webhooks.go index 6e064b195..8d123c0bf 100644 --- a/webhooks/virtualmachinesetresourcepolicy/webhooks.go +++ b/webhooks/virtualmachinesetresourcepolicy/webhooks.go @@ -7,9 +7,14 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinesetresourcepolicy/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinesetresourcepolicy/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + return v1alpha2.AddToManager(ctx, mgr) + } return v1alpha1.AddToManager(ctx, mgr) } From 865f40785535962fc4cbc157bec1291cb1c90daa Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Thu, 28 Sep 2023 14:45:53 -0500 Subject: [PATCH 14/54] Hook up ctrl-runtime webhooks when the v1a2 FFS is enabled This is used for the version conversion wehbooks. This can also support validation and mutation webhooks but our own webhook framework predates ctrl-runtime support for that so that is an improvement for later. --- main.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/main.go b/main.go index c40785abc..65958e50e 100644 --- a/main.go +++ b/main.go @@ -18,9 +18,12 @@ import ( klog "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/controllers" "github.com/vmware-tanzu/vm-operator/pkg" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/manager" "github.com/vmware-tanzu/vm-operator/webhooks" @@ -265,6 +268,12 @@ func main() { return err } + if lib.IsVMServiceV1Alpha2FSSEnabled() { + if err := addConversionWebhooksToManager(ctx, mgr); err != nil { + return err + } + } + return webhooks.AddToManager(ctx, mgr) } @@ -296,6 +305,57 @@ func main() { } } +// addConversionWebhooksToManager adds the ctrl-runtime managed webhooks. We just use these +// for version conversion, but they can also do mutation and validation webhook callbacks +// instead of our separate webhooks. +func addConversionWebhooksToManager(_ *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if err := (&v1alpha1.VirtualMachine{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineClass{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.ClusterVirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachinePublishRequest{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineService{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha1.VirtualMachineSetResourcePolicy{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + + if err := (&v1alpha2.VirtualMachine{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineClass{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.ClusterVirtualMachineImage{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachinePublishRequest{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineService{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + if err := (&v1alpha2.VirtualMachineSetResourcePolicy{}).SetupWebhookWithManager(mgr); err != nil { + return err + } + + return nil +} + func configureWebhookTLS(opts *webhook.Options) { tlsCfgFunc := func(cfg *tls.Config) { cfg.MinVersion = tls.VersionTLS12 From 7ab36cf698eda58374b24bacb7da4e7755f5765e Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Fri, 29 Sep 2023 13:41:25 -0700 Subject: [PATCH 15/54] Make v1a1 webconsole controller v1a2 VMProvider aware (#234) When v1a2 FSS gets enabled, only the v1a2 VM Provider will be available for use. Since v1a1 webconsole CRs are still supported in v1a2 world, the v1a1 VM Provider that it uses will NPE. This requires the v1a1 controller to be v1a2 Provider aware. --- .../v1alpha1/webconsolerequest_controller.go | 41 ++++++--- .../v1alpha1/webconsolerequest_intg_test.go | 86 ++++++++++++++----- .../v1alpha1/webconsolerequest_suite_test.go | 6 +- .../v1alpha1/webconsolerequest_unit_test.go | 48 ++++++++++- 4 files changed, 148 insertions(+), 33 deletions(-) diff --git a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_controller.go b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_controller.go index de46589eb..52b34ca56 100644 --- a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_controller.go +++ b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_controller.go @@ -21,7 +21,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + vmopv1alpha2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/patch" "github.com/vmware-tanzu/vm-operator/pkg/record" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider" @@ -50,6 +52,7 @@ func AddToManager(ctx *context.ControllerManagerContext, mgr manager.Manager) er ctrl.Log.WithName("controllers").WithName(controlledTypeName), record.New(mgr.GetEventRecorderFor(controllerNameLong)), ctx.VMProvider, + ctx.VMProviderA2, ) return ctrl.NewControllerManagedBy(mgr). @@ -62,21 +65,24 @@ func NewReconciler( client client.Client, logger logr.Logger, recorder record.Recorder, - vmProvider vmprovider.VirtualMachineProviderInterface) *Reconciler { + vmProvider vmprovider.VirtualMachineProviderInterface, + vmProviderA2 vmprovider.VirtualMachineProviderInterfaceA2) *Reconciler { return &Reconciler{ - Client: client, - Logger: logger, - Recorder: recorder, - VMProvider: vmProvider, + Client: client, + Logger: logger, + Recorder: recorder, + VMProvider: vmProvider, + VMProviderA2: vmProviderA2, } } // Reconciler reconciles a WebConsoleRequest object. type Reconciler struct { client.Client - Logger logr.Logger - Recorder record.Recorder - VMProvider vmprovider.VirtualMachineProviderInterface + Logger logr.Logger + Recorder record.Recorder + VMProvider vmprovider.VirtualMachineProviderInterface + VMProviderA2 vmprovider.VirtualMachineProviderInterfaceA2 } // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=webconsolerequests,verbs=get;list;watch;create;update;patch;delete @@ -164,10 +170,23 @@ func (r *Reconciler) ReconcileNormal(ctx *context.WebConsoleRequestContext) erro ctx.Logger.Info("Finished reconciling WebConsoleRequest") }() - ticket, err := r.VMProvider.GetVirtualMachineWebMKSTicket(ctx, ctx.VM, ctx.WebConsoleRequest.Spec.PublicKey) - if err != nil { - return errors.Wrapf(err, "failed to get webmksticket") + var ticket string + var err error + if lib.IsVMServiceV1Alpha2FSSEnabled() { + v1a2VM := &vmopv1alpha2.VirtualMachine{} + _ = ctx.VM.ConvertTo(v1a2VM) + + ticket, err = r.VMProviderA2.GetVirtualMachineWebMKSTicket(ctx, v1a2VM, ctx.WebConsoleRequest.Spec.PublicKey) + if err != nil { + return errors.Wrapf(err, "failed to get webmksticket") + } + } else { + ticket, err = r.VMProvider.GetVirtualMachineWebMKSTicket(ctx, ctx.VM, ctx.WebConsoleRequest.Spec.PublicKey) + if err != nil { + return errors.Wrapf(err, "failed to get webmksticket") + } } + r.Recorder.EmitEvent(ctx.WebConsoleRequest, "Acquired Ticket", nil, false) ctx.WebConsoleRequest.Status.Response = ticket diff --git a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_intg_test.go b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_intg_test.go index d691aa843..20a55f0f3 100644 --- a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_intg_test.go +++ b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_intg_test.go @@ -5,6 +5,7 @@ package v1alpha1_test import ( "context" + "os" "time" . "github.com/onsi/ginkgo" @@ -16,7 +17,9 @@ import ( "k8s.io/apimachinery/pkg/types" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + vmopv1alpha2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" webconsolerequest "github.com/vmware-tanzu/vm-operator/controllers/virtualmachinewebconsolerequest/v1alpha1" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/test/builder" ) @@ -26,10 +29,12 @@ func intgTests() { func webConsoleRequestReconcile() { var ( - ctx *builder.IntegrationTestContext - wcr *vmopv1.WebConsoleRequest - vm *vmopv1.VirtualMachine - proxySvc *corev1.Service + ctx *builder.IntegrationTestContext + wcr *vmopv1.WebConsoleRequest + vm *vmopv1.VirtualMachine + proxySvc *corev1.Service + v1a1ProviderCalled bool + v1a2ProviderCalled bool ) getWebConsoleRequest := func(ctx *builder.IntegrationTestContext, objKey types.NamespacedName) *vmopv1.WebConsoleRequest { @@ -84,21 +89,32 @@ func webConsoleRequestReconcile() { fakeVMProvider.Lock() defer fakeVMProvider.Unlock() + fakeVMProvider.GetVirtualMachineWebMKSTicketFn = func(ctx context.Context, vm *vmopv1.VirtualMachine, pubKey string) (string, error) { + v1a1ProviderCalled = true return "some-fake-webmksticket", nil } + + fakeVMProviderA2.Lock() + defer fakeVMProviderA2.Unlock() + + fakeVMProviderA2.GetVirtualMachineWebMKSTicketFn = func(ctx context.Context, vm *vmopv1alpha2.VirtualMachine, pubKey string) (string, error) { + v1a2ProviderCalled = true + return "some-fake-webmksticket-1", nil + } }) AfterEach(func() { ctx.AfterEach() ctx = nil fakeVMProvider.Reset() + fakeVMProviderA2.Reset() + v1a1ProviderCalled = false + v1a2ProviderCalled = false }) Context("Reconcile", func() { - BeforeEach(func() { - Expect(ctx.Client.Create(ctx, vm)).To(Succeed()) - Expect(ctx.Client.Create(ctx, wcr)).To(Succeed()) + JustBeforeEach(func() { Expect(ctx.Client.Create(ctx, proxySvc)).To(Succeed()) proxySvc.Status = corev1.ServiceStatus{ LoadBalancer: corev1.LoadBalancerStatus{ @@ -110,27 +126,57 @@ func webConsoleRequestReconcile() { }, } Expect(ctx.Client.Status().Update(ctx, proxySvc)).To(Succeed()) + Expect(ctx.Client.Create(ctx, vm)).To(Succeed()) + Expect(ctx.Client.Create(ctx, wcr)).To(Succeed()) }) - AfterEach(func() { + JustAfterEach(func() { err := ctx.Client.Delete(ctx, wcr) Expect(err == nil || k8serrors.IsNotFound(err)).To(BeTrue()) err = ctx.Client.Delete(ctx, vm) Expect(err == nil || k8serrors.IsNotFound(err)).To(BeTrue()) + err = ctx.Client.Delete(ctx, proxySvc) + Expect(err == nil || k8serrors.IsNotFound(err)).To(BeTrue()) + }) + + When("v1a1 provider", func() { + It("resource successfully created", func() { + Eventually(func(g Gomega) { + wcr = getWebConsoleRequest(ctx, types.NamespacedName{Name: wcr.Name, Namespace: wcr.Namespace}) + g.Expect(wcr).ToNot(BeNil()) + g.Expect(wcr.Status.Response).ToNot(BeEmpty()) + }).Should(Succeed(), "waiting for webconsolerequest to be") + + Expect(v1a1ProviderCalled).Should(BeTrue()) + Expect(wcr.Status.ProxyAddr).To(Equal("192.168.0.1")) + Expect(wcr.Status.Response).ToNot(BeEmpty()) + Expect(wcr.Status.ExpiryTime.Time).To(BeTemporally("~", time.Now(), webconsolerequest.DefaultExpiryTime)) + Expect(wcr.Labels).To(HaveKeyWithValue(webconsolerequest.UUIDLabelKey, string(wcr.UID))) + }) }) - It("resource successfully created", func() { - Eventually(func() bool { - wcr = getWebConsoleRequest(ctx, types.NamespacedName{Name: wcr.Name, Namespace: wcr.Namespace}) - if wcr != nil && wcr.Status.Response != "" { - return true - } - return false - }).Should(BeTrue(), "waiting for webconsolerequest to be") - Expect(wcr.Status.ProxyAddr).To(Equal("192.168.0.1")) - Expect(wcr.Status.Response).ToNot(BeEmpty()) - Expect(wcr.Status.ExpiryTime.Time).To(BeTemporally("~", time.Now(), webconsolerequest.DefaultExpiryTime)) - Expect(wcr.Labels).To(HaveKeyWithValue(webconsolerequest.UUIDLabelKey, string(wcr.UID))) + When("VM Service v1a2 FSS enabled", func() { + BeforeEach(func() { + Expect(os.Setenv(lib.VMServiceV1Alpha2FSS, lib.TrueString)).To(Succeed()) + }) + + AfterEach(func() { + Expect(os.Unsetenv(lib.VMServiceV1Alpha2FSS)).To(Succeed()) + }) + + It("resource successfully created", func() { + Eventually(func(g Gomega) { + wcr = getWebConsoleRequest(ctx, types.NamespacedName{Name: wcr.Name, Namespace: wcr.Namespace}) + g.Expect(wcr).ToNot(BeNil()) + g.Expect(wcr.Status.Response).ToNot(BeEmpty()) + }).Should(Succeed(), "waiting for webconsolerequest to be") + + Expect(v1a2ProviderCalled).Should(BeTrue()) + Expect(wcr.Status.ProxyAddr).To(Equal("192.168.0.1")) + Expect(wcr.Status.Response).ToNot(BeEmpty()) + Expect(wcr.Status.ExpiryTime.Time).To(BeTemporally("~", time.Now(), webconsolerequest.DefaultExpiryTime)) + Expect(wcr.Labels).To(HaveKeyWithValue(webconsolerequest.UUIDLabelKey, string(wcr.UID))) + }) }) }) } diff --git a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_suite_test.go b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_suite_test.go index 48d5e73f6..9a3bc0776 100644 --- a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_suite_test.go +++ b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_suite_test.go @@ -16,12 +16,16 @@ import ( "github.com/vmware-tanzu/vm-operator/test/builder" ) -var fakeVMProvider = providerfake.NewVMProvider() +var ( + fakeVMProvider = providerfake.NewVMProvider() + fakeVMProviderA2 = providerfake.NewVMProviderA2() +) var suite = builder.NewTestSuiteForController( v1alpha1.AddToManager, func(ctx *ctrlContext.ControllerManagerContext, _ ctrlmgr.Manager) error { ctx.VMProvider = fakeVMProvider + ctx.VMProviderA2 = fakeVMProviderA2 return nil }, ) diff --git a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_unit_test.go b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_unit_test.go index 3fe3f8dfe..2bfc74b88 100644 --- a/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_unit_test.go +++ b/controllers/virtualmachinewebconsolerequest/v1alpha1/webconsolerequest_unit_test.go @@ -5,6 +5,7 @@ package v1alpha1_test import ( "context" + "os" "time" . "github.com/onsi/ginkgo" @@ -15,8 +16,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + vmopv1alpha2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" webconsolerequest "github.com/vmware-tanzu/vm-operator/controllers/virtualmachinewebconsolerequest/v1alpha1" vmopContext "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" providerfake "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/fake" "github.com/vmware-tanzu/vm-operator/test/builder" ) @@ -79,8 +82,10 @@ func unitTestsReconcile() { ctx.Logger, ctx.Recorder, ctx.VMProvider, + ctx.VMProviderA2, ) fakeVMProvider = ctx.VMProvider.(*providerfake.VMProvider) + fakeVMProviderA2 = ctx.VMProviderA2.(*providerfake.VMProviderA2) wcrCtx = &vmopContext.WebConsoleRequestContext{ Context: ctx, @@ -95,18 +100,35 @@ func unitTestsReconcile() { ctx = nil initObjects = nil reconciler = nil - fakeVMProvider.Reset() }) Context("ReconcileNormal", func() { + var ( + v1a1ProviderCalled bool + v1a2ProviderCalled bool + ) + BeforeEach(func() { initObjects = append(initObjects, wcr, vm, proxySvc) }) JustBeforeEach(func() { fakeVMProvider.GetVirtualMachineWebMKSTicketFn = func(ctx context.Context, vm *vmopv1.VirtualMachine, pubKey string) (string, error) { + v1a1ProviderCalled = true return "some-fake-webmksticket", nil } + + fakeVMProviderA2.GetVirtualMachineWebMKSTicketFn = func(ctx context.Context, vm *vmopv1alpha2.VirtualMachine, pubKey string) (string, error) { + v1a2ProviderCalled = true + return "some-fake-webmksticket-1", nil + } + }) + + AfterEach(func() { + fakeVMProvider.Reset() + fakeVMProviderA2.Reset() + v1a1ProviderCalled = false + v1a2ProviderCalled = false }) When("NoOp", func() { @@ -114,6 +136,7 @@ func unitTestsReconcile() { err := reconciler.ReconcileNormal(wcrCtx) Expect(err).ToNot(HaveOccurred()) + Expect(v1a1ProviderCalled).Should(BeTrue()) Expect(wcrCtx.WebConsoleRequest.Status.ProxyAddr).To(Equal("dummy-proxy-ip")) Expect(wcrCtx.WebConsoleRequest.Status.Response).ToNot(BeEmpty()) Expect(wcrCtx.WebConsoleRequest.Status.ExpiryTime.Time).To(BeTemporally("~", time.Now(), webconsolerequest.DefaultExpiryTime)) @@ -121,5 +144,28 @@ func unitTestsReconcile() { Expect(wcrCtx.WebConsoleRequest.Labels).To(HaveKey(webconsolerequest.UUIDLabelKey)) }) }) + + When("VM Service v1alpha2 FSS is enabled", func() { + BeforeEach(func() { + Expect(os.Setenv(lib.VMServiceV1Alpha2FSS, lib.TrueString)).To(Succeed()) + }) + + AfterEach(func() { + Expect(os.Unsetenv(lib.VMServiceV1Alpha2FSS)).To(Succeed()) + }) + + It("returns success", func() { + err := reconciler.ReconcileNormal(wcrCtx) + Expect(err).ToNot(HaveOccurred()) + + Expect(v1a2ProviderCalled).Should(BeTrue()) + Expect(wcrCtx.WebConsoleRequest.Status.ProxyAddr).To(Equal("dummy-proxy-ip")) + Expect(wcrCtx.WebConsoleRequest.Status.Response).ToNot(BeEmpty()) + Expect(wcrCtx.WebConsoleRequest.Status.ExpiryTime.Time).To(BeTemporally("~", time.Now(), webconsolerequest.DefaultExpiryTime)) + // Checking the label key only because UID will not be set to a resource during unit test. + Expect(wcrCtx.WebConsoleRequest.Labels).To(HaveKey(webconsolerequest.UUIDLabelKey)) + }) + }) + }) } From eac14d23186098fa484fb92e01f2e64bcab4b54c Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Fri, 29 Sep 2023 17:14:24 -0400 Subject: [PATCH 16/54] Persist VM disk path and type info in ExtraConfig (#229) This patch persists a VM's disk path and type (determined by the existence of VDiskID) in ExtraConfig. This is required to register any disks as FCDs and expose them as PVCs during a restore operation. --- .../providers/vsphere/constants/constants.go | 3 + .../vsphere/virtualmachine/backup.go | 66 ++++++++++- .../vsphere/virtualmachine/backup_test.go | 111 +++++++++--------- 3 files changed, 117 insertions(+), 63 deletions(-) diff --git a/pkg/vmprovider/providers/vsphere/constants/constants.go b/pkg/vmprovider/providers/vsphere/constants/constants.go index a37a6367b..be7b042cf 100644 --- a/pkg/vmprovider/providers/vsphere/constants/constants.go +++ b/pkg/vmprovider/providers/vsphere/constants/constants.go @@ -125,4 +125,7 @@ const ( // BackupVMBootstrapDataExtraConfigKey is the ExtraConfig key to the VM's // bootstrap data object, compressed using gzip and base64-encoded. BackupVMBootstrapDataExtraConfigKey = "vmservice.virtualmachine.bootstrapdata" + // BackupVMDiskDataExtraConfigKey is the ExtraConfig key to the VM's disk info + // data in JSON, compressed using gzip and base64-encoded. + BackupVMDiskDataExtraConfigKey = "vmservice.virtualmachine.diskdata" ) diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go b/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go index 5ee3dd20e..96b71d747 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go @@ -4,6 +4,9 @@ package virtualmachine import ( + goctx "context" + "encoding/json" + "sigs.k8s.io/yaml" "github.com/vmware/govmomi/object" @@ -13,11 +16,21 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/util" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" + res "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/resources" ) -// BackupVirtualMachine backs up the VM's kube data and bootstrap data into the -// vSphere VM's ExtraConfig. The status fields will not be backed up, as we expect -// to recreate and reconcile these resources during the restore process. +type VMDiskData struct { + // ID of the virtual disk object (only set for FCDs). + VDiskID string + // Filename contains the datastore path to the virtual disk. + FileName string +} + +// BackupVirtualMachine backs up the required data of a VM into its ExtraConfig. +// Currently, the following data is backed up: +// - Kubernetes VirtualMachine object in YAML format (without its .status field). +// - VM bootstrap data in JSON (if provided). +// - List of VM disk data in JSON (including FCDs attached to the VM). func BackupVirtualMachine( vmCtx context.VirtualMachineContext, vcVM *object.VirtualMachine, @@ -37,6 +50,12 @@ func BackupVirtualMachine( vmCtx.Logger.Info("No VM bootstrap data is provided for backup") } + vmDiskData, err := getEncodedVMDiskData(vmCtx, vcVM) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get encoded VM disk data") + return err + } + _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ ExtraConfig: []types.BaseOptionValue{ &types.OptionValue{ @@ -47,6 +66,10 @@ func BackupVirtualMachine( Key: constants.BackupVMBootstrapDataExtraConfigKey, Value: vmBootstrapData, }, + &types.OptionValue{ + Key: constants.BackupVMDiskDataExtraConfigKey, + Value: vmDiskData, + }, }}) if err != nil { vmCtx.Logger.Error(err, "Failed to reconfigure VM ExtraConfig for backup") @@ -72,10 +95,43 @@ func getEncodedVMBootstrapData(bootstrapData map[string]string) (string, error) return "", nil } - bootstrapDataYaml, err := yaml.Marshal(bootstrapData) + bootstrapDataJSON, err := json.Marshal(bootstrapData) + if err != nil { + return "", err + } + + return util.EncodeGzipBase64(string(bootstrapDataJSON)) +} + +func getEncodedVMDiskData( + ctx goctx.Context, vcVM *object.VirtualMachine) (string, error) { + resVM := res.NewVMFromObject(vcVM) + disks, err := resVM.GetVirtualDisks(ctx) + if err != nil { + return "", err + } + + diskData := []VMDiskData{} + for _, device := range disks { + if disk, ok := device.(*types.VirtualDisk); ok { + vmDiskData := VMDiskData{} + if disk.VDiskId != nil { + vmDiskData.VDiskID = disk.VDiskId.Id + } + if b, ok := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok { + vmDiskData.FileName = b.FileName + } + // Only add the disk data if it's not empty. + if vmDiskData != (VMDiskData{}) { + diskData = append(diskData, vmDiskData) + } + } + } + + diskDataJSON, err := json.Marshal(diskData) if err != nil { return "", err } - return util.EncodeGzipBase64(string(bootstrapDataYaml)) + return util.EncodeGzipBase64(string(diskDataJSON)) } diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go b/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go index 5d60d3332..dfa40ca18 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go @@ -4,6 +4,8 @@ package virtualmachine_test import ( + "encoding/json" + "sigs.k8s.io/yaml" . "github.com/onsi/ginkgo" @@ -15,16 +17,16 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/util" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/session" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/virtualmachine" "github.com/vmware-tanzu/vm-operator/test/builder" ) func backupTests() { var ( - ctx *builder.TestContextForVCSim - vcVM *object.VirtualMachine - vmCtx context.VirtualMachineContext - bootstrapData map[string]string + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + vmCtx context.VirtualMachineContext ) BeforeEach(func() { @@ -32,12 +34,12 @@ func backupTests() { var err error vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") - Expect(err).ToNot(HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) vmCtx = context.VirtualMachineContext{ Context: ctx, Logger: suite.GetLogger().WithValues("vmName", vcVM.Name()), - VM: builder.DummyVirtualMachine(), + VM: &vmopv1.VirtualMachine{}, } }) @@ -46,78 +48,71 @@ func backupTests() { ctx = nil }) - Context("VM bootstrap data is NOT provided", func() { + Context("VM Kube data", func() { + + BeforeEach(func() { + vmCtx.VM = builder.DummyVirtualMachine() + }) - It("Should back up only VM kube data in ExtraConfig", func() { + It("Should backup VM kube data YAML without status field in ExtraConfig", func() { Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, vmCtx, nil) + + vmCopy := vmCtx.VM.DeepCopy() + vmCopy.Status = vmopv1.VirtualMachineStatus{} + vmCopyYaml, err := yaml.Marshal(vmCopy) + Expect(err).NotTo(HaveOccurred()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, string(vmCopyYaml)) }) }) - Context("VM bootstrap data is provided", func() { + Context("VM bootstrap data", func() { - BeforeEach(func() { - bootstrapData = map[string]string{"foo": "bar"} - }) + It("Should back up bootstrap data as JSON in ExtraConfig", func() { + bootstrapData := map[string]string{"foo": "bar"} + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, bootstrapData)).To(Succeed()) - AfterEach(func() { - bootstrapData = nil + bootstrapDataJSON, err := json.Marshal(bootstrapData) + Expect(err).NotTo(HaveOccurred()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMBootstrapDataExtraConfigKey, string(bootstrapDataJSON)) }) + }) - It("Should back up both VM kube data and bootstrap data in ExtraConfig", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, bootstrapData)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, vmCtx, bootstrapData) + Context("VM Disk data", func() { + + It("Should backup VM disk data as JSON in ExtraConfig", func() { + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + + // Use the default disk info from the vcSim VM for testing. + diskData := []virtualmachine.VMDiskData{ + { + VDiskID: "", + FileName: "[LocalDS_0] DC0_C0_RP0_VM0/disk1.vmdk", + }, + } + diskDataJSON, err := json.Marshal(diskData) + Expect(err).NotTo(HaveOccurred()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, string(diskDataJSON)) }) }) } -// verifyBackupDataInExtraConfig verifies that the backup data is stored in VM's ExtraConfig. -// It gets the expected data and compares it with the actual data in ExtraConfig. func verifyBackupDataInExtraConfig( ctx *builder.TestContextForVCSim, vcVM *object.VirtualMachine, - vmCtx context.VirtualMachineContext, - bootstrapData map[string]string) { - // Get expected backup VM's kube data from the VM CR. - vmCopy := vmCtx.VM.DeepCopy() - vmCopy.Status = vmopv1.VirtualMachineStatus{} - vmCopyYaml, err := yaml.Marshal(vmCopy) - Expect(err).NotTo(HaveOccurred()) - Expect(vmCopyYaml).NotTo(BeEmpty()) - expectedKubeData, err := util.EncodeGzipBase64(string(vmCopyYaml)) - Expect(err).NotTo(HaveOccurred()) - - var expectedBootstrapData string - if bootstrapData != nil { - bootstrapDataYaml, err := yaml.Marshal(bootstrapData) - Expect(err).NotTo(HaveOccurred()) - expectedBootstrapData, err = util.EncodeGzipBase64(string(bootstrapDataYaml)) - Expect(err).NotTo(HaveOccurred()) - } + expectedKey, expectedValDecoded string) { - // Get actual backup data from VM's ExtraConfig. + // Get the VM's ExtraConfig and convert it to map. moID := vcVM.Reference().Value objVM := ctx.GetVMFromMoID(moID) - Expect(objVM).ToNot(BeNil()) + Expect(objVM).NotTo(BeNil()) var moVM mo.VirtualMachine Expect(objVM.Properties(ctx, objVM.Reference(), []string{"config.extraConfig"}, &moVM)).To(Succeed()) + ecMap := session.ExtraConfigToMap(moVM.Config.ExtraConfig) - // Compare the backup data in ExtraConfig with the expected data. - var kubeDataMatched, bootstrapDataMatched bool - for _, ec := range moVM.Config.ExtraConfig { - if ec.GetOptionValue().Key == constants.BackupVMKubeDataExtraConfigKey { - Expect(ec.GetOptionValue().Value.(string)).To(Equal(expectedKubeData)) - kubeDataMatched = true - } else if ec.GetOptionValue().Key == constants.BackupVMBootstrapDataExtraConfigKey { - Expect(ec.GetOptionValue().Value.(string)).To(Equal(expectedBootstrapData)) - bootstrapDataMatched = true - } - - if kubeDataMatched && bootstrapDataMatched { - return - } - } - - Expect(kubeDataMatched).To(BeTrue(), "Encoded VM kube data is not found in VM's ExtraConfig") - Expect(bootstrapDataMatched).To(BeTrue(), "Encoded VM bootstrap data is not found in VM's ExtraConfig") + // Verify the expected key exists in ExtraConfig and the decoded values match. + Expect(ecMap).To(HaveKey(expectedKey)) + ecValRaw := ecMap[expectedKey] + ecValDecoded, err := util.TryToDecodeBase64Gzip([]byte(ecValRaw)) + Expect(err).NotTo(HaveOccurred()) + Expect(ecValDecoded).To(Equal(expectedValDecoded)) } From 7253b3cf1837ead6afc55335839f45fc1bb7e705 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Mon, 2 Oct 2023 14:08:48 -0500 Subject: [PATCH 17/54] Fix a few typos in the v1a2 bootstrap types --- api/v1alpha2/common/types.go | 2 +- api/v1alpha2/sysprep/sysprep.go | 2 +- config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/v1alpha2/common/types.go b/api/v1alpha2/common/types.go index a6f168890..a21b3f1d3 100644 --- a/api/v1alpha2/common/types.go +++ b/api/v1alpha2/common/types.go @@ -70,7 +70,7 @@ type KeyValuePair struct { } // KeyValueOrSecretKeySelectorPair is useful when wanting to realize a map as a -// list of key/value pairs where each value could also referenced data stored in +// list of key/value pairs where each value could also reference data stored in // a Secret resource. type KeyValueOrSecretKeySelectorPair struct { // Key is the key part of the key/value pair. diff --git a/api/v1alpha2/sysprep/sysprep.go b/api/v1alpha2/sysprep/sysprep.go index 7a6260989..5a2c00ded 100644 --- a/api/v1alpha2/sysprep/sysprep.go +++ b/api/v1alpha2/sysprep/sysprep.go @@ -49,7 +49,7 @@ type GUIRunOnce struct { // customization. // // +optional - Commands []string `json:"commmands,omitempty"` + Commands []string `json:"commands,omitempty"` } // GUIUnattended maps to the GuiUnattended key in the sysprep.xml answer file. diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml index 3a5ca2047..06c3acfbd 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml @@ -1241,7 +1241,7 @@ spec: description: GUIRunOnce is a representation of the Sysprep GuiRunOnce key. properties: - commmands: + commands: description: Commands is a list of commands to run at first user logon, after guest customization. items: @@ -1446,8 +1446,8 @@ spec: items: description: KeyValueOrSecretKeySelectorPair is useful when wanting to realize a map as a list of key/value pairs - where each value could also referenced data stored in - a Secret resource. + where each value could also reference data stored in a + Secret resource. properties: key: description: Key is the key part of the key/value pair. From e78e962076fbfc586b814c6f72de5f5068ddd30f Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Tue, 3 Oct 2023 16:04:48 -0400 Subject: [PATCH 18/54] =?UTF-8?q?=F0=9F=8C=B1=20Prevent=20non-admin=20user?= =?UTF-8?q?s=20from=20setting=20first-boot-done=20annotation=20(#237)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **What does this PR do, and why is it needed?** This patch updates the VM mutation webhook to disallow non-admin users from adding/updating/removing the `first-boot-done` annotation. It is only meant to be modified by the VM-Operator. Also, it fixes the current implementation which could cause all VM updates from non-admin users to fail if the `cloud-init-instance-id` annotations exist. **Which issue(s) is/are addressed by this PR?** Fixes #221 **Are there any special notes for your reviewer**: This change is a prerequisite for another upcoming change that will rely on the existence of the `first-boot-done` annotation set by the VM-Operator. **Please add a release note if necessary**: ```release-note Prevent non-admin users from setting first-boot-done annotation ``` --- api/v1alpha1/virtualmachine_types.go | 5 ++ api/v1alpha2/virtualmachine_types.go | 5 ++ .../vsphere/session/session_vm_update.go | 6 +- .../providers/vsphere/vmprovider_vm.go | 6 +- .../vsphere2/session/session_vm_update.go | 6 +- .../providers/vsphere2/vmprovider_vm.go | 6 +- .../validation/virtualmachine_validator.go | 23 ++++--- .../virtualmachine_validator_unit_test.go | 62 ++++++++++++++---- .../validation/virtualmachine_validator.go | 23 ++++--- .../virtualmachine_validator_unit_test.go | 63 +++++++++++++++---- 10 files changed, 146 insertions(+), 59 deletions(-) diff --git a/api/v1alpha1/virtualmachine_types.go b/api/v1alpha1/virtualmachine_types.go index f54e66943..051efe096 100644 --- a/api/v1alpha1/virtualmachine_types.go +++ b/api/v1alpha1/virtualmachine_types.go @@ -105,6 +105,11 @@ const ( // the same with the first boot Instance ID to prevent Cloud-Init from treating this VM as first boot // due to different Instance ID. This annotation is used in upgrade script. InstanceIDAnnotation = GroupName + "/cloud-init-instance-id" + + // FirstBootDoneAnnotation is an annotation that indicates the VM has been + // booted at least once. This annotation cannot be set by users and will not + // be removed once set until the VM is deleted. + FirstBootDoneAnnotation = "virtualmachine." + GroupName + "/first-boot-done" ) // VirtualMachinePort is unused and can be considered deprecated. diff --git a/api/v1alpha2/virtualmachine_types.go b/api/v1alpha2/virtualmachine_types.go index 8d40ef6d0..90af22976 100644 --- a/api/v1alpha2/virtualmachine_types.go +++ b/api/v1alpha2/virtualmachine_types.go @@ -102,6 +102,11 @@ const ( // the same with the first boot Instance ID to prevent Cloud-Init from treating this VM as first boot // due to different Instance ID. This annotation is used in upgrade script. InstanceIDAnnotation = GroupName + "/cloud-init-instance-id" + + // FirstBootDoneAnnotation is an annotation that indicates the VM has been + // booted at least once. This annotation cannot be set by users and will not + // be removed once set until the VM is deleted. + FirstBootDoneAnnotation = "virtualmachine." + GroupName + "/first-boot-done" ) // VirtualMachinePowerState defines a VM's desired and observed power states. diff --git a/pkg/vmprovider/providers/vsphere/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere/session/session_vm_update.go index 8e03dab3a..8626c6d34 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere/session/session_vm_update.go @@ -32,10 +32,6 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/virtualmachine" ) -const ( - FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" -) - type VMMetadata struct { Data map[string]string Transport vmopv1.VirtualMachineMetadataTransport @@ -930,7 +926,7 @@ func (s *Session) UpdateVirtualMachine( if vmCtx.VM.Annotations == nil { vmCtx.VM.Annotations = map[string]string{} } - vmCtx.VM.Annotations[FirstBootDoneAnnotation] = "true" + vmCtx.VM.Annotations[vmopv1.FirstBootDoneAnnotation] = "true" } return nil } diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm.go index 86f92d439..15ce347da 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm.go @@ -41,10 +41,6 @@ type vmCreateArgs = session.VMCreateArgs type vmUpdateArgs = session.VMUpdateArgs type vmMetadata = session.VMMetadata -const ( - FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" -) - var ( createCountLock sync.Mutex concurrentCreateCount int @@ -811,7 +807,7 @@ func (vs *vSphereVMProvider) vmUpdateGetArgs( } func isVMFirstBoot(vmCtx context.VirtualMachineContext) bool { - if _, ok := vmCtx.VM.Annotations[FirstBootDoneAnnotation]; ok { + if _, ok := vmCtx.VM.Annotations[vmopv1.FirstBootDoneAnnotation]; ok { return false } diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go index 1154c20fa..81cef4856 100644 --- a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go @@ -31,10 +31,6 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" ) -const ( - FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" -) - // VMUpdateArgs contains the arguments needed to update a VM on VC. type VMUpdateArgs struct { VMClass *vmopv1.VirtualMachineClass @@ -951,7 +947,7 @@ func (s *Session) UpdateVirtualMachine( if vmCtx.VM.Annotations == nil { vmCtx.VM.Annotations = map[string]string{} } - vmCtx.VM.Annotations[FirstBootDoneAnnotation] = "true" + vmCtx.VM.Annotations[vmopv1.FirstBootDoneAnnotation] = "true" } return nil } diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go index fb7721812..1ef1c92a5 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go @@ -62,10 +62,6 @@ type VMCreateArgs struct { // TODO: Until we sort out what the Session becomes. type vmUpdateArgs = session.VMUpdateArgs -const ( - FirstBootDoneAnnotation = "virtualmachine.vmoperator.vmware.com/first-boot-done" -) - var ( createCountLock sync.Mutex concurrentCreateCount int @@ -1161,7 +1157,7 @@ func (vs *vSphereVMProvider) vmUpdateGetArgs( } func isVMFirstBoot(vmCtx context.VirtualMachineContextA2) bool { - if _, ok := vmCtx.VM.Annotations[FirstBootDoneAnnotation]; ok { + if _, ok := vmCtx.VM.Annotations[vmopv1.FirstBootDoneAnnotation]; ok { return false } diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go index 6babc8d51..30ea6d026 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go @@ -64,7 +64,7 @@ const ( invalidNextRestartTimeOnCreate = "cannot restart VM on create" invalidNextRestartTimeOnUpdate = "must be formatted as RFC3339Nano" invalidNextRestartTimeOnUpdateNow = "mutation webhooks are required to restart VM" - settingAnnotationNotAllowed = "adding this annotation is not allowed" + modifyAnnotationNotAllowedForNonAdmin = "modifying this annotation is not allowed for non-admin users" ) // +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachine,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,versions=v1alpha1,name=default.validating.virtualmachine.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 @@ -120,7 +120,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. fieldErrs = append(fieldErrs, v.validateInstanceStorageVolumes(ctx, vm, nil)...) fieldErrs = append(fieldErrs, v.validatePowerStateOnCreate(ctx, vm)...) fieldErrs = append(fieldErrs, v.validateNextRestartTimeOnCreate(ctx, vm)...) - fieldErrs = append(fieldErrs, v.validateAnnotation(ctx, vm)...) + fieldErrs = append(fieldErrs, v.validateAnnotation(ctx, vm, nil)...) validationErrs := make([]string, 0, len(fieldErrs)) for _, fieldErr := range fieldErrs { @@ -180,7 +180,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. fieldErrs = append(fieldErrs, v.validateReadinessProbe(ctx, vm)...) fieldErrs = append(fieldErrs, v.validateInstanceStorageVolumes(ctx, vm, oldVM)...) fieldErrs = append(fieldErrs, v.validateNextRestartTimeOnUpdate(ctx, vm, oldVM)...) - fieldErrs = append(fieldErrs, v.validateAnnotation(ctx, vm)...) + fieldErrs = append(fieldErrs, v.validateAnnotation(ctx, vm, oldVM)...) validationErrs := make([]string, 0, len(fieldErrs)) for _, fieldErr := range fieldErrs { @@ -742,17 +742,26 @@ func (v validator) isNetworkRestrictedForReadinessProbe(ctx *context.WebhookRequ return configMap.Data[isRestrictedNetworkKey] == "true", nil } -func (v validator) validateAnnotation(ctx *context.WebhookRequestContext, vm *vmopv1.VirtualMachine) field.ErrorList { +func (v validator) validateAnnotation(ctx *context.WebhookRequestContext, vm, oldVM *vmopv1.VirtualMachine) field.ErrorList { var allErrs field.ErrorList - if ctx.IsPrivilegedAccount || vm.Annotations == nil { + if ctx.IsPrivilegedAccount { return allErrs } + // Use an empty VM if the oldVM is nil to validate a creation request. + if oldVM == nil { + oldVM = &vmopv1.VirtualMachine{} + } + annotationPath := field.NewPath("metadata", "annotations") - if _, exists := vm.Annotations[vmopv1.InstanceIDAnnotation]; exists { - allErrs = append(allErrs, field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), settingAnnotationNotAllowed)) + if vm.Annotations[vmopv1.InstanceIDAnnotation] != oldVM.Annotations[vmopv1.InstanceIDAnnotation] { + allErrs = append(allErrs, field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), modifyAnnotationNotAllowedForNonAdmin)) + } + + if vm.Annotations[vmopv1.FirstBootDoneAnnotation] != oldVM.Annotations[vmopv1.FirstBootDoneAnnotation] { + allErrs = append(allErrs, field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), modifyAnnotationNotAllowedForNonAdmin)) } return allErrs diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go index 19fda5408..8db901a3a 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go @@ -6,6 +6,7 @@ package validation_test import ( "fmt" "os" + "strings" "time" . "github.com/onsi/ginkgo" @@ -35,6 +36,8 @@ const ( updateSuffix = "-updated" dummyNamespaceImageName = "dummy-namespace-image" dummyClusterImageName = "dummy-cluster-image" + dummyInstanceIDVal = "dummy-instance-id" + dummyFirstBootDoneVal = "dummy-first-boot-done" ) func unitTests() { @@ -221,7 +224,7 @@ func unitTestsValidateCreate() { isSysprepTransportUsed bool powerState vmopv1.VirtualMachinePowerState nextRestartTime string - isForbiddenAnnotation bool + adminOnlyAnnotations bool } validateCreate := func(args createArgs, expectedAllowed bool, expectedReason string, expectedErr error) { @@ -369,8 +372,9 @@ func unitTestsValidateCreate() { ctx.vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName } - if args.isForbiddenAnnotation { - ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = "some-value" + if args.adminOnlyAnnotations { + ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = dummyInstanceIDVal + ctx.vm.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal } ctx.vm.Spec.PowerState = args.powerState @@ -512,9 +516,13 @@ func unitTestsValidateCreate() { Entry("should disallow creating VM with non-empty, invalid nextRestartTime value", createArgs{nextRestartTime: "hello"}, false, field.Invalid(nextRestartTimePath, "hello", "cannot restart VM on create").Error(), nil), - Entry("should deny cloud-init-instance-id annotation set by SSO user", createArgs{isForbiddenAnnotation: true}, false, - field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "adding this annotation is not allowed").Error(), nil), - Entry("should allow cloud-init-instance-id annotation set by service user", createArgs{isServiceUser: true, isForbiddenAnnotation: true}, true, nil, nil), + + Entry("should disallow creating VM with admin-only annotations set by SSO user", createArgs{adminOnlyAnnotations: true}, false, + strings.Join([]string{ + field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + }, ", "), nil), + Entry("should allow creating VM with admin-only annotations set by service user", createArgs{isServiceUser: true, adminOnlyAnnotations: true}, true, nil, nil), ) } @@ -542,7 +550,9 @@ func unitTestsValidateUpdate() { newPowerStateEmptyAllowed bool nextRestartTime string lastRestartTime string - isForbiddenAnnotation bool + addAdminOnlyAnnotations bool + updateAdminOnlyAnnotations bool + removeAdminOnlyAnnotations bool } validateUpdate := func(args updateArgs, expectedAllowed bool, expectedReason string, expectedErr error) { @@ -591,9 +601,21 @@ func unitTestsValidateUpdate() { ctx.oldVM.Spec.NextRestartTime = args.lastRestartTime ctx.vm.Spec.NextRestartTime = args.nextRestartTime - if args.isForbiddenAnnotation { - ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = "some-value" + if args.addAdminOnlyAnnotations { + ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = dummyInstanceIDVal + ctx.vm.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal + } + if args.updateAdminOnlyAnnotations { + ctx.oldVM.Annotations[vmopv1.InstanceIDAnnotation] = dummyInstanceIDVal + ctx.oldVM.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal + ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = dummyInstanceIDVal + updateSuffix + ctx.vm.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal + updateSuffix + } + if args.removeAdminOnlyAnnotations { + ctx.oldVM.Annotations[vmopv1.InstanceIDAnnotation] = dummyInstanceIDVal + ctx.oldVM.Annotations[vmopv1.FirstBootDoneAnnotation] = updateSuffix } + // Named network provider undoNamedNetProvider := initNamedNetworkProviderConfig( ctx, @@ -686,9 +708,25 @@ func unitTestsValidateUpdate() { Entry("should disallow updating VM with non-empty, invalid nextRestartTime value ", updateArgs{nextRestartTime: "hello"}, false, field.Invalid(nextRestartTimePath, "hello", "must be formatted as RFC3339Nano").Error(), nil), - Entry("should deny cloud-init-instance-id annotation set by SSO user", updateArgs{isForbiddenAnnotation: true}, false, - field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "adding this annotation is not allowed").Error(), nil), - Entry("should allow cloud-init-instance-id annotation set by service user", updateArgs{isServiceUser: true, isForbiddenAnnotation: true}, true, nil, nil), + + Entry("should disallow adding admin-only annotations by SSO user", updateArgs{addAdminOnlyAnnotations: true}, false, + strings.Join([]string{ + field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + }, ", "), nil), + Entry("should disallow updating admin-only annotations by SSO user", updateArgs{updateAdminOnlyAnnotations: true}, false, + strings.Join([]string{ + field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + }, ", "), nil), + Entry("should disallow removing admin-only annotations by SSO user", updateArgs{removeAdminOnlyAnnotations: true}, false, + strings.Join([]string{ + field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + }, ", "), nil), + Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, addAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, updateAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, removeAdminOnlyAnnotations: true}, true, nil, nil), ) When("the update is performed while object deletion", func() { diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index 2b5412bd1..fb537c01c 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -59,7 +59,7 @@ const ( invalidNextRestartTimeOnCreate = "cannot restart VM on create" invalidNextRestartTimeOnUpdate = "must be formatted as RFC3339Nano" invalidNextRestartTimeOnUpdateNow = "mutation webhooks are required to restart VM" - settingAnnotationNotAllowed = "adding this annotation is not allowed" + modifyAnnotationNotAllowedForNonAdmin = "modifying this annotation is not allowed for non-admin users" ) // +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha2-virtualmachine,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,versions=v1alpha2,name=default.validating.virtualmachine.v1alpha2.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 @@ -115,7 +115,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. fieldErrs = append(fieldErrs, v.validateAdvanced(ctx, vm)...) fieldErrs = append(fieldErrs, v.validatePowerStateOnCreate(ctx, vm)...) fieldErrs = append(fieldErrs, v.validateNextRestartTimeOnCreate(ctx, vm)...) - fieldErrs = append(fieldErrs, v.validateAnnotation(ctx, vm)...) + fieldErrs = append(fieldErrs, v.validateAnnotation(ctx, vm, nil)...) validationErrs := make([]string, 0, len(fieldErrs)) for _, fieldErr := range fieldErrs { @@ -170,7 +170,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. fieldErrs = append(fieldErrs, v.validateAdvanced(ctx, vm)...) fieldErrs = append(fieldErrs, v.validateInstanceStorageVolumes(ctx, vm, oldVM)...) fieldErrs = append(fieldErrs, v.validateNextRestartTimeOnUpdate(ctx, vm, oldVM)...) - fieldErrs = append(fieldErrs, v.validateAnnotation(ctx, vm)...) + fieldErrs = append(fieldErrs, v.validateAnnotation(ctx, vm, oldVM)...) validationErrs := make([]string, 0, len(fieldErrs)) for _, fieldErr := range fieldErrs { @@ -683,17 +683,26 @@ func (v validator) isNetworkRestrictedForReadinessProbe(ctx *context.WebhookRequ return configMap.Data[isRestrictedNetworkKey] == "true", nil } -func (v validator) validateAnnotation(ctx *context.WebhookRequestContext, vm *vmopv1.VirtualMachine) field.ErrorList { +func (v validator) validateAnnotation(ctx *context.WebhookRequestContext, vm, oldVM *vmopv1.VirtualMachine) field.ErrorList { var allErrs field.ErrorList - if ctx.IsPrivilegedAccount || vm.Annotations == nil { + if ctx.IsPrivilegedAccount { return allErrs } + // Use an empty VM if the oldVM is nil to validate a creation request. + if oldVM == nil { + oldVM = &vmopv1.VirtualMachine{} + } + annotationPath := field.NewPath("metadata", "annotations") - if _, exists := vm.Annotations[vmopv1.InstanceIDAnnotation]; exists { - allErrs = append(allErrs, field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), settingAnnotationNotAllowed)) + if vm.Annotations[vmopv1.InstanceIDAnnotation] != oldVM.Annotations[vmopv1.InstanceIDAnnotation] { + allErrs = append(allErrs, field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), modifyAnnotationNotAllowedForNonAdmin)) + } + + if vm.Annotations[vmopv1.FirstBootDoneAnnotation] != oldVM.Annotations[vmopv1.FirstBootDoneAnnotation] { + allErrs = append(allErrs, field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), modifyAnnotationNotAllowedForNonAdmin)) } return allErrs diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go index 92da314c4..4bb3c797c 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go @@ -6,6 +6,7 @@ package validation_test import ( "fmt" "os" + "strings" "time" . "github.com/onsi/ginkgo" @@ -30,7 +31,9 @@ import ( ) const ( - updateSuffix = "-updated" + updateSuffix = "-updated" + dummyInstanceIDVal = "dummy-instance-id" + dummyFirstBootDoneVal = "dummy-first-boot-done" ) func unitTests() { @@ -107,7 +110,7 @@ func unitTestsValidateCreate() { isBootstrapVAppConfigInline bool powerState vmopv1.VirtualMachinePowerState nextRestartTime string - isForbiddenAnnotation bool + adminOnlyAnnotations bool } validateCreate := func(args createArgs, expectedAllowed bool, expectedReason string, expectedErr error) { @@ -251,8 +254,9 @@ func unitTestsValidateCreate() { } } - if args.isForbiddenAnnotation { - ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = "some-value" + if args.adminOnlyAnnotations { + ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = updateSuffix + ctx.vm.Annotations[vmopv1.FirstBootDoneAnnotation] = updateSuffix } ctx.vm.Spec.PowerState = args.powerState @@ -368,9 +372,13 @@ func unitTestsValidateCreate() { Entry("should disallow creating VM with non-empty, invalid nextRestartTime value", createArgs{nextRestartTime: "hello"}, false, field.Invalid(nextRestartTimePath, "hello", "cannot restart VM on create").Error(), nil), - Entry("should deny cloud-init-instance-id annotation set by user", createArgs{isForbiddenAnnotation: true}, false, - field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "adding this annotation is not allowed").Error(), nil), - Entry("should allow cloud-init-instance-id annotation set by service user", createArgs{isServiceUser: true, isForbiddenAnnotation: true}, true, nil, nil), + + Entry("should disallow creating VM with admin-only annotations set by SSO user", createArgs{adminOnlyAnnotations: true}, false, + strings.Join([]string{ + field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + }, ", "), nil), + Entry("should allow creating VM with admin-only annotations set by service user", createArgs{isServiceUser: true, adminOnlyAnnotations: true}, true, nil, nil), ) } @@ -396,7 +404,9 @@ func unitTestsValidateUpdate() { newPowerStateEmptyAllowed bool nextRestartTime string lastRestartTime string - isForbiddenAnnotation bool + addAdminOnlyAnnotations bool + updateAdminOnlyAnnotations bool + removeAdminOnlyAnnotations bool } validateUpdate := func(args updateArgs, expectedAllowed bool, expectedReason string, expectedErr error) { @@ -450,8 +460,19 @@ func unitTestsValidateUpdate() { ctx.vm.Spec.PowerState = args.newPowerState } - if args.isForbiddenAnnotation { - ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = "some-value" + if args.addAdminOnlyAnnotations { + ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = dummyInstanceIDVal + ctx.vm.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal + } + if args.updateAdminOnlyAnnotations { + ctx.oldVM.Annotations[vmopv1.InstanceIDAnnotation] = dummyInstanceIDVal + ctx.oldVM.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal + ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = dummyInstanceIDVal + updateSuffix + ctx.vm.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal + updateSuffix + } + if args.removeAdminOnlyAnnotations { + ctx.oldVM.Annotations[vmopv1.InstanceIDAnnotation] = dummyInstanceIDVal + ctx.oldVM.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal } ctx.oldVM.Spec.NextRestartTime = args.lastRestartTime @@ -532,9 +553,25 @@ func unitTestsValidateUpdate() { Entry("should disallow updating VM with non-empty, invalid nextRestartTime value ", updateArgs{nextRestartTime: "hello"}, false, field.Invalid(nextRestartTimePath, "hello", "must be formatted as RFC3339Nano").Error(), nil), - Entry("should deny cloud-init-instance-id annotation set by user", updateArgs{isForbiddenAnnotation: true}, false, - field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "adding this annotation is not allowed").Error(), nil), - Entry("should allow cloud-init-instance-id annotation set by service user", updateArgs{isServiceUser: true, isForbiddenAnnotation: true}, true, nil, nil), + + Entry("should disallow adding admin-only annotations by SSO user", updateArgs{addAdminOnlyAnnotations: true}, false, + strings.Join([]string{ + field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + }, ", "), nil), + Entry("should disallow updating admin-only annotations by SSO user", updateArgs{updateAdminOnlyAnnotations: true}, false, + strings.Join([]string{ + field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + }, ", "), nil), + Entry("should disallow removing admin-only annotations by SSO user", updateArgs{removeAdminOnlyAnnotations: true}, false, + strings.Join([]string{ + field.Forbidden(annotationPath.Child(vmopv1.InstanceIDAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), + }, ", "), nil), + Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, addAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, updateAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, removeAdminOnlyAnnotations: true}, true, nil, nil), ) When("the update is performed while object deletion", func() { From 69e9728b353443db8947a71008ded02505ce8da8 Mon Sep 17 00:00:00 2001 From: Yiyi Zhou <91219164+zyiyi11@users.noreply.github.com> Date: Wed, 4 Oct 2023 11:16:39 -0700 Subject: [PATCH 19/54] Return err when unmarshaling ovf file to trigger reconcile (#235) --- .../vsphere/contentlibrary/content_library_provider.go | 7 ++----- .../vsphere/contentlibrary/content_library_test.go | 6 +++--- .../vsphere2/contentlibrary/content_library_provider.go | 3 ++- .../vsphere2/contentlibrary/content_library_test.go | 6 +++--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_provider.go b/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_provider.go index 918ce920e..f67fff270 100644 --- a/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_provider.go +++ b/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_provider.go @@ -196,10 +196,11 @@ func (cs *provider) RetrieveOvfEnvelopeFromLibraryItem(ctx context.Context, item _ = downloadedFileContent.Close() }() + // OVF file is validated during upload, err here can be internet error. envelope, err := ovf.Unmarshal(downloadedFileContent) if err != nil { logger.Error(err, "error parsing the OVF envelope") - return nil, nil + return nil, err } return envelope, nil @@ -319,10 +320,6 @@ func (cs *provider) VirtualMachineImageResourceForLibrary(ctx context.Context, logger.Error(err, "error extracting the OVF envelope from the library item", "itemName", item.Name) return nil, err } - if ovfEnvelope == nil { - logger.Error(err, "no valid OVF envelope found, skipping library item", "itemName", item.Name) - return nil, nil - } case library.ItemTypeVMTX: // Do not try to populate VMTX types, but resVm.GetOvfProperties() should return an // OvfEnvelope. diff --git a/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_test.go b/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_test.go index 7d2a28563..5903da7dd 100644 --- a/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_test.go +++ b/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_test.go @@ -136,7 +136,7 @@ func clTests() { }) }) - Context("called with an OVF that is invalid", func() { + Context("called with an OVF that is invalid because of network connectivity issue", func() { var ovfPath string AfterEach(func() { @@ -145,7 +145,7 @@ func clTests() { } }) - It("does not return error", func() { + It("returns error", func() { ovf, err := os.CreateTemp("", "fake-*.ovf") Expect(err).NotTo(HaveOccurred()) ovfPath = ovf.Name() @@ -169,7 +169,7 @@ func clTests() { Expect(libItem2.Name).To(Equal(libItem.Name)) ovfEnvelope, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem2) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(HaveOccurred()) Expect(ovfEnvelope).To(BeNil()) }) }) diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go index be2ff212d..b6b66e8c6 100644 --- a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_provider.go @@ -212,10 +212,11 @@ func (cs *provider) RetrieveOvfEnvelopeFromLibraryItem(ctx context.Context, item _ = downloadedFileContent.Close() }() + // OVF file is validated during upload, err here can be internet error. envelope, err := ovf.Unmarshal(downloadedFileContent) if err != nil { logger.Error(err, "error parsing the OVF envelope") - return nil, nil + return nil, err } return envelope, nil diff --git a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go index 968eac039..286f3037b 100644 --- a/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go +++ b/pkg/vmprovider/providers/vsphere2/contentlibrary/content_library_test.go @@ -98,7 +98,7 @@ func clTests() { }) }) - Context("called with an OVF that is invalid", func() { + Context("called with an OVF that is invalid because of network connectivity issue", func() { var ovfPath string AfterEach(func() { @@ -107,7 +107,7 @@ func clTests() { } }) - It("does not return error", func() { + It("returns error", func() { ovf, err := os.CreateTemp("", "fake-*.ovf") Expect(err).NotTo(HaveOccurred()) ovfPath = ovf.Name() @@ -131,7 +131,7 @@ func clTests() { Expect(libItem2.Name).To(Equal(libItem.Name)) ovfEnvelope, err := clProvider.RetrieveOvfEnvelopeFromLibraryItem(ctx, libItem2) - Expect(err).ToNot(HaveOccurred()) + Expect(err).To(HaveOccurred()) Expect(ovfEnvelope).To(BeNil()) }) }) From 913e8fa18a1df54874717d0d2470ba7c1ad345a8 Mon Sep 17 00:00:00 2001 From: Arunesh Pandey Date: Wed, 4 Oct 2023 14:52:11 -0700 Subject: [PATCH 20/54] Emit a webhook warning when OvfEnv transport is specified (#231) This change emits a warning when a user specifies OvfEnv transport in a VM's spec. In order to support warnings, modify the BuildValidationResponse so it can now take warnings from any validations. At some point, we should move our webhooks to controller-runtime generated webhooks which will remove the need to synthesize the response from errors and warnings ourselves. --- webhooks/common/response.go | 3 +- webhooks/common/response_test.go | 18 +++- .../persistentvolumeclaim_validator.go | 12 +-- .../validation/virtualmachine_validator.go | 37 +++++-- .../virtualmachine_validator_unit_test.go | 97 +++++++++++++++++++ .../validation/virtualmachine_validator.go | 4 +- .../virtualmachineclass_validator.go | 4 +- .../virtualmachineclass_validator.go | 4 +- .../virtualmachinepublishrequest_validator.go | 6 +- .../virtualmachinepublishrequest_validator.go | 6 +- .../virtualmachineservice_validator.go | 4 +- .../virtualmachineservice_validator.go | 4 +- ...rtualmachinesetresourcepolicy_validator.go | 4 +- ...rtualmachinesetresourcepolicy_validator.go | 4 +- .../validation/webconsolerequest_validator.go | 4 +- 15 files changed, 171 insertions(+), 40 deletions(-) diff --git a/webhooks/common/response.go b/webhooks/common/response.go index a3302a62b..ea64801de 100644 --- a/webhooks/common/response.go +++ b/webhooks/common/response.go @@ -19,6 +19,7 @@ import ( // errors returned attempting to validate the ingress data. func BuildValidationResponse( ctx *context.WebhookRequestContext, + validationWarnings admission.Warnings, validationErrs []string, err error, additionalValidationErrors ...string) (response admission.Response) { @@ -76,5 +77,5 @@ func BuildValidationResponse( } } - return admission.Allowed("") + return admission.Allowed("").WithWarnings(validationWarnings...) } diff --git a/webhooks/common/response_test.go b/webhooks/common/response_test.go index d54a080ff..62d058a7a 100644 --- a/webhooks/common/response_test.go +++ b/webhooks/common/response_test.go @@ -35,7 +35,7 @@ var _ = Describe("Validation Response", func() { When("No errors occur", func() { It("Returns allowed", func() { - response := common.BuildValidationResponse(ctx, nil, nil) + response := common.BuildValidationResponse(ctx, nil, nil, nil) Expect(response.Allowed).To(BeTrue()) }) }) @@ -43,7 +43,7 @@ var _ = Describe("Validation Response", func() { When("Validation errors occur", func() { It("Returns denied", func() { validationErrs := []string{"this is required"} - response := common.BuildValidationResponse(ctx, validationErrs, nil) + response := common.BuildValidationResponse(ctx, nil, validationErrs, nil) Expect(response.Allowed).To(BeFalse()) Expect(response.Result).ToNot(BeNil()) Expect(response.Result.Code).To(Equal(int32(http.StatusUnprocessableEntity))) @@ -51,10 +51,20 @@ var _ = Describe("Validation Response", func() { }) }) - Context("Returns denied for expected well-known errors", func() { + When("Validation has warnings", func() { + It("Returns allowed, with warnings", func() { + validationWarnings := []string{"this is deprecated"} + response := common.BuildValidationResponse(ctx, validationWarnings, nil, nil) + Expect(response.Allowed).To(BeTrue()) + Expect(response.Warnings).To(Equal(validationWarnings)) + Expect(response.Result).ToNot(BeNil()) + Expect(response.Result.Code).To(Equal(int32(http.StatusOK))) + }) + }) + Context("Returns denied for expected well-known errors", func() { wellKnownError := func(err error, expectedCode int) { - response := common.BuildValidationResponse(ctx, nil, err) + response := common.BuildValidationResponse(ctx, nil, nil, err) Expect(response.Allowed).To(BeFalse()) Expect(response.Result).ToNot(BeNil()) Expect(response.Result.Code).To(Equal(int32(expectedCode))) diff --git a/webhooks/persistentvolumeclaim/validation/persistentvolumeclaim_validator.go b/webhooks/persistentvolumeclaim/validation/persistentvolumeclaim_validator.go index 1bb380aa5..a2648813a 100644 --- a/webhooks/persistentvolumeclaim/validation/persistentvolumeclaim_validator.go +++ b/webhooks/persistentvolumeclaim/validation/persistentvolumeclaim_validator.go @@ -78,7 +78,7 @@ func (v validator) For() schema.GroupVersionKind { func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission.Response { if isPrivilegedAccountForISPVC(ctx) { - return common.BuildValidationResponse(ctx, nil, nil) + return common.BuildValidationResponse(ctx, nil, nil, nil) } var fieldErrs field.ErrorList @@ -87,12 +87,12 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. fmt.Sprintf(operationNotAllowedOnPVC, admissionv1.Create))) } - return common.BuildValidationResponse(ctx, convertToStringArray(fieldErrs), nil) + return common.BuildValidationResponse(ctx, nil, convertToStringArray(fieldErrs), nil) } func (v validator) ValidateDelete(ctx *context.WebhookRequestContext) admission.Response { if isPrivilegedAccountForISPVC(ctx) { - return common.BuildValidationResponse(ctx, nil, nil) + return common.BuildValidationResponse(ctx, nil, nil, nil) } var fieldErrs field.ErrorList @@ -101,12 +101,12 @@ func (v validator) ValidateDelete(ctx *context.WebhookRequestContext) admission. fmt.Sprintf(operationNotAllowedOnPVC, admissionv1.Delete))) } - return common.BuildValidationResponse(ctx, convertToStringArray(fieldErrs), nil) + return common.BuildValidationResponse(ctx, nil, convertToStringArray(fieldErrs), nil) } func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission.Response { if isPrivilegedAccountForISPVC(ctx) { - return common.BuildValidationResponse(ctx, nil, nil) + return common.BuildValidationResponse(ctx, nil, nil, nil) } var fieldErrs field.ErrorList // If instance storage labels already exists for resource, do not allow update resource @@ -117,7 +117,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. fieldErrs = append(fieldErrs, field.Forbidden(labelPath, addingISLabelNotAllowed)) } - return common.BuildValidationResponse(ctx, convertToStringArray(fieldErrs), nil) + return common.BuildValidationResponse(ctx, nil, convertToStringArray(fieldErrs), nil) } // isInstanceStorageLabelPresent - returns true/false depending on presence of instance storage label. diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go index 30ea6d026..775dc01d6 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go @@ -65,6 +65,8 @@ const ( invalidNextRestartTimeOnUpdate = "must be formatted as RFC3339Nano" invalidNextRestartTimeOnUpdateNow = "mutation webhooks are required to restart VM" modifyAnnotationNotAllowedForNonAdmin = "modifying this annotation is not allowed for non-admin users" + OVFEnvTransportDeprecated = "OvfEnv transport is deprecated. Please use CloudInit or vAppConfig transport instead." + ExtraConfigTransportDeprecated = "ExtraConfig transport is deprecated. Please use CloudInit or vAppConfig transport instead." ) // +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha1-virtualmachine,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,versions=v1alpha1,name=default.validating.virtualmachine.v1alpha1.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 @@ -107,8 +109,12 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. } var fieldErrs field.ErrorList + var validationWarnings admission.Warnings + + warnings, errs := v.validateMetadata(ctx, vm) + fieldErrs = append(fieldErrs, errs...) + validationWarnings = append(validationWarnings, warnings...) - fieldErrs = append(fieldErrs, v.validateMetadata(ctx, vm)...) fieldErrs = append(fieldErrs, v.validateAvailabilityZone(ctx, vm, nil)...) fieldErrs = append(fieldErrs, v.validateImage(ctx, vm)...) fieldErrs = append(fieldErrs, v.validateClass(ctx, vm)...) @@ -127,7 +133,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, validationWarnings, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -162,6 +168,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. } var fieldErrs field.ErrorList + var validationWarnings admission.Warnings // Check if an immutable field has been modified. fieldErrs = append(fieldErrs, v.validateImmutableFields(ctx, vm, oldVM)...) @@ -172,7 +179,10 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. // Validations for allowed updates. Return validation responses here for // conditional updates regardless of whether the update is allowed or not. - fieldErrs = append(fieldErrs, v.validateMetadata(ctx, vm)...) + warnings, errs := v.validateMetadata(ctx, vm) + fieldErrs = append(fieldErrs, errs...) + validationWarnings = append(validationWarnings, warnings...) + fieldErrs = append(fieldErrs, v.validateAvailabilityZone(ctx, vm, oldVM)...) fieldErrs = append(fieldErrs, v.validateNetwork(ctx, vm)...) fieldErrs = append(fieldErrs, v.validateVolumes(ctx, vm)...) @@ -187,18 +197,31 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, validationWarnings, validationErrs, nil) } -func (v validator) validateMetadata(ctx *context.WebhookRequestContext, vm *vmopv1.VirtualMachine) field.ErrorList { +func (v validator) validateMetadata(ctx *context.WebhookRequestContext, vm *vmopv1.VirtualMachine) (admission.Warnings, field.ErrorList) { var allErrs field.ErrorList + var allWarnings admission.Warnings if vm.Spec.VmMetadata == nil { - return allErrs + return allWarnings, allErrs } mdPath := field.NewPath("spec", "vmMetadata") + // OvfEnv transport is marked as deprecated. Emit a warnings to indicate that to users. + if vm.Spec.VmMetadata.Transport == vmopv1.VirtualMachineMetadataOvfEnvTransport { + allWarnings = append(allWarnings, + fmt.Sprint(OVFEnvTransportDeprecated)) + } + + // ExtraConfig transport is marked as deprecated. Emit a warnings to indicate that to users. + if vm.Spec.VmMetadata.Transport == vmopv1.VirtualMachineMetadataExtraConfigTransport { + allWarnings = append(allWarnings, + fmt.Sprint(ExtraConfigTransportDeprecated)) + } + if vm.Spec.VmMetadata.ConfigMapName != "" && vm.Spec.VmMetadata.SecretName != "" { allErrs = append(allErrs, field.Invalid(mdPath.Child("configMapName"), vm.Spec.VmMetadata.ConfigMapName, fmt.Sprintf(metadataTransportResourcesInvalid, mdPath.Child("configMapName"), mdPath.Child("secretName")))) @@ -213,7 +236,7 @@ func (v validator) validateMetadata(ctx *context.WebhookRequestContext, vm *vmop } } - return allErrs + return allWarnings, allErrs } func (v validator) validateImage(ctx *context.WebhookRequestContext, vm *vmopv1.VirtualMachine) field.ErrorList { diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go index 8db901a3a..75bb111b4 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go @@ -30,6 +30,7 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/config" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/network" "github.com/vmware-tanzu/vm-operator/test/builder" + vmopv1validation "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha1/validation" ) const ( @@ -42,7 +43,9 @@ const ( func unitTests() { Describe("Invoking ValidateCreate", unitTestsValidateCreate) + Describe("Invoking ValidateCreate for Webhook Warnings", unitTestsValidateCreateWarnings) Describe("Invoking ValidateUpdate", unitTestsValidateUpdate) + Describe("Invoking ValidateUpdate for Webhook Warnings", unitTestsValidateUpdateWarnings) Describe("Invoking ValidateDelete", unitTestsValidateDelete) } @@ -526,6 +529,100 @@ func unitTestsValidateCreate() { ) } +func unitTestsValidateCreateWarnings() { + var ( + ctx *unitValidatingWebhookContext + ) + + Context("VM Metadata transport deprecation warnings", func() { + var err error + BeforeEach(func() { + ctx = newUnitTestContextForValidatingWebhook(false) + }) + + When("OvfEnv transport is specified in VM Metadata", func() { + BeforeEach(func() { + ctx.vm.Spec.VmMetadata.Transport = vmopv1.VirtualMachineMetadataOvfEnvTransport + ctx.WebhookRequestContext.Obj, err = builder.ToUnstructured(ctx.vm) + Expect(err).ToNot(HaveOccurred()) + }) + It("the request should succeed, but with a warning", func() { + response := ctx.ValidateCreate(&ctx.WebhookRequestContext) + Expect(response.Allowed).To(Equal(true)) + Expect(response.Warnings).To(Equal([]string{vmopv1validation.OVFEnvTransportDeprecated})) + }) + }) + When("ExtraConfig transport is specified in VM Metadata", func() { + BeforeEach(func() { + ctx.vm.Spec.VmMetadata.Transport = vmopv1.VirtualMachineMetadataExtraConfigTransport + ctx.WebhookRequestContext.Obj, err = builder.ToUnstructured(ctx.vm) + Expect(err).ToNot(HaveOccurred()) + }) + It("the request should succeed, but with a warning", func() { + response := ctx.ValidateCreate(&ctx.WebhookRequestContext) + Expect(response.Allowed).To(Equal(true)) + + Expect(response.Warnings).To(Equal([]string{vmopv1validation.ExtraConfigTransportDeprecated})) + }) + }) + }) +} + +func unitTestsValidateUpdateWarnings() { + var ( + ctx *unitValidatingWebhookContext + ) + + Context("VM Metadata transport deprecation warnings", func() { + var err error + BeforeEach(func() { + ctx = newUnitTestContextForValidatingWebhook(true) + }) + AfterEach(func() { + ctx = nil + }) + + When("OvfEnv transport is specified in VM Metadata", func() { + BeforeEach(func() { + // Updates to metadata are only allowed in powered off state. + ctx.vm.Spec.PowerState = vmopv1.VirtualMachinePoweredOff + ctx.vm.Spec.VmMetadata.Transport = vmopv1.VirtualMachineMetadataOvfEnvTransport + + ctx.WebhookRequestContext.Obj, err = builder.ToUnstructured(ctx.vm) + Expect(err).ToNot(HaveOccurred()) + ctx.WebhookRequestContext.OldObj, err = builder.ToUnstructured(ctx.oldVM) + Expect(err).ToNot(HaveOccurred()) + + }) + It("the request should succeed, but with a warning", func() { + response := ctx.ValidateUpdate(&ctx.WebhookRequestContext) + Expect(response.Allowed).To(Equal(true)) + Expect(response.Warnings).To(Equal([]string{vmopv1validation.OVFEnvTransportDeprecated})) + }) + }) + When("ExtraConfig transport is specified in VM Metadata", func() { + BeforeEach(func() { + + // Updates to metadata are only allowed in powered off state. + ctx.vm.Spec.PowerState = vmopv1.VirtualMachinePoweredOff + ctx.vm.Spec.VmMetadata.Transport = vmopv1.VirtualMachineMetadataExtraConfigTransport + + ctx.WebhookRequestContext.Obj, err = builder.ToUnstructured(ctx.vm) + Expect(err).ToNot(HaveOccurred()) + ctx.WebhookRequestContext.OldObj, err = builder.ToUnstructured(ctx.oldVM) + Expect(err).ToNot(HaveOccurred()) + + }) + It("the request should succeed, but with a warning", func() { + response := ctx.ValidateUpdate(&ctx.WebhookRequestContext) + Expect(response.Allowed).To(Equal(true)) + + Expect(response.Warnings).To(Equal([]string{vmopv1validation.ExtraConfigTransportDeprecated})) + }) + }) + }) +} + func unitTestsValidateUpdate() { var ( ctx *unitValidatingWebhookContext diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index fb537c01c..ae13bf987 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -122,7 +122,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -177,7 +177,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validateBootstrap( diff --git a/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator.go b/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator.go index 2d815aa79..77b17522c 100644 --- a/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator.go +++ b/webhooks/virtualmachineclass/v1alpha1/validation/virtualmachineclass_validator.go @@ -78,7 +78,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -92,7 +92,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validatePolicies(ctx *context.WebhookRequestContext, vmClass *vmopv1.VirtualMachineClass, diff --git a/webhooks/virtualmachineclass/v1alpha2/validation/virtualmachineclass_validator.go b/webhooks/virtualmachineclass/v1alpha2/validation/virtualmachineclass_validator.go index 6b6bc9154..14aee640d 100644 --- a/webhooks/virtualmachineclass/v1alpha2/validation/virtualmachineclass_validator.go +++ b/webhooks/virtualmachineclass/v1alpha2/validation/virtualmachineclass_validator.go @@ -78,7 +78,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -92,7 +92,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validatePolicies(ctx *context.WebhookRequestContext, vmClass *vmopv1.VirtualMachineClass, diff --git a/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator.go b/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator.go index bf3e65c7b..37f401087 100644 --- a/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator.go +++ b/webhooks/virtualmachinepublishrequest/v1alpha1/validation/virtualmachinepublishrequest_validator.go @@ -65,7 +65,7 @@ func (v validator) For() schema.GroupVersionKind { func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission.Response { if !lib.IsWCPVMImageRegistryEnabled() { - return common.BuildValidationResponse(ctx, []string{"WCP_VM_Image_Registry feature not enabled"}, nil) + return common.BuildValidationResponse(ctx, nil, []string{"WCP_VM_Image_Registry feature not enabled"}, nil) } vmpub, err := v.vmPublishRequestFromUnstructured(ctx.Obj) @@ -83,7 +83,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -111,7 +111,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validateSource(ctx *context.WebhookRequestContext, vmpub *vmopv1.VirtualMachinePublishRequest) field.ErrorList { diff --git a/webhooks/virtualmachinepublishrequest/v1alpha2/validation/virtualmachinepublishrequest_validator.go b/webhooks/virtualmachinepublishrequest/v1alpha2/validation/virtualmachinepublishrequest_validator.go index 1791d7aea..28310bf1d 100644 --- a/webhooks/virtualmachinepublishrequest/v1alpha2/validation/virtualmachinepublishrequest_validator.go +++ b/webhooks/virtualmachinepublishrequest/v1alpha2/validation/virtualmachinepublishrequest_validator.go @@ -66,7 +66,7 @@ func (v validator) For() schema.GroupVersionKind { func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission.Response { if !lib.IsWCPVMImageRegistryEnabled() { - return common.BuildValidationResponse(ctx, []string{"WCP_VM_Image_Registry feature not enabled"}, nil) + return common.BuildValidationResponse(ctx, nil, []string{"WCP_VM_Image_Registry feature not enabled"}, nil) } vmpub, err := v.vmPublishRequestFromUnstructured(ctx.Obj) @@ -84,7 +84,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -112,7 +112,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validateSource(ctx *context.WebhookRequestContext, vmpub *vmopv1.VirtualMachinePublishRequest) field.ErrorList { diff --git a/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator.go b/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator.go index d701348c6..a2b7538e9 100644 --- a/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator.go +++ b/webhooks/virtualmachineservice/v1alpha1/validation/virtualmachineservice_validator.go @@ -97,7 +97,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -124,7 +124,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validateMetadata(ctx *context.WebhookRequestContext, vmService *vmopv1.VirtualMachineService) field.ErrorList { diff --git a/webhooks/virtualmachineservice/v1alpha2/validation/virtualmachineservice_validator.go b/webhooks/virtualmachineservice/v1alpha2/validation/virtualmachineservice_validator.go index b2387c537..e98751803 100644 --- a/webhooks/virtualmachineservice/v1alpha2/validation/virtualmachineservice_validator.go +++ b/webhooks/virtualmachineservice/v1alpha2/validation/virtualmachineservice_validator.go @@ -97,7 +97,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -124,7 +124,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validateMetadata(ctx *context.WebhookRequestContext, vmService *vmopv1.VirtualMachineService) field.ErrorList { diff --git a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator.go b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator.go index 8ba2636a3..87790524d 100644 --- a/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator.go +++ b/webhooks/virtualmachinesetresourcepolicy/v1alpha1/validation/virtualmachinesetresourcepolicy_validator.go @@ -75,7 +75,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -101,7 +101,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validateSpec(ctx *context.WebhookRequestContext, vmRP *vmopv1.VirtualMachineSetResourcePolicy) field.ErrorList { diff --git a/webhooks/virtualmachinesetresourcepolicy/v1alpha2/validation/virtualmachinesetresourcepolicy_validator.go b/webhooks/virtualmachinesetresourcepolicy/v1alpha2/validation/virtualmachinesetresourcepolicy_validator.go index 3ebc60c82..0c6ff6453 100644 --- a/webhooks/virtualmachinesetresourcepolicy/v1alpha2/validation/virtualmachinesetresourcepolicy_validator.go +++ b/webhooks/virtualmachinesetresourcepolicy/v1alpha2/validation/virtualmachinesetresourcepolicy_validator.go @@ -75,7 +75,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -101,7 +101,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validateSpec(ctx *context.WebhookRequestContext, vmRP *vmopv1.VirtualMachineSetResourcePolicy) field.ErrorList { diff --git a/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator.go b/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator.go index 921e90c04..636951171 100644 --- a/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator.go +++ b/webhooks/virtualmachinewebconsolerequest/v1alpha1/validation/webconsolerequest_validator.go @@ -75,7 +75,7 @@ func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission. validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { @@ -101,7 +101,7 @@ func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission. for _, fieldErr := range fieldErrs { validationErrs = append(validationErrs, fieldErr.Error()) } - return common.BuildValidationResponse(ctx, validationErrs, nil) + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) } func (v validator) validateMetadata(ctx *context.WebhookRequestContext, wcr *vmopv1.WebConsoleRequest) field.ErrorList { From be66239ac8ce586f2f218685ef9cb5359ad9f4e2 Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Wed, 4 Oct 2023 14:54:31 -0700 Subject: [PATCH 21/54] Reverse configSpec.firmware order of precedence to Image then Classes (#238) This change reverses of the order of precedence to populate the configSpec from the image first then the class. This is necessary with vsphere UI unable to create VM Classes with an empty firmware, potentially creating scenarios for VM boot failures when the wrong firmware type is populated. The image is known to always have the supported firmware type for boot. This reversal can be changed back once vSphere GUI supports creating VM Classes with an optional firmware type for a user. --- .../vsphere/virtualmachine/configspec.go | 8 +++-- .../vsphere/virtualmachine/configspec_test.go | 12 +++++++ .../vsphere2/virtualmachine/configspec.go | 10 ++++-- .../virtualmachine/configspec_test.go | 36 ++++++++++++++++++- 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/configspec.go b/pkg/vmprovider/providers/vsphere/virtualmachine/configspec.go index 22c92c58f..7ff2d6ab6 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/configspec.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/configspec.go @@ -88,8 +88,12 @@ func CreateConfigSpec( } } - // Use firmware type from the image if config spec doesn't have it. - if configSpec.Firmware == "" && imageFirmware != "" { + // Always use the image's firmware type if present. + // This is necessary until the vSphere UI can support creating VM Classes with + // an empty/nil firmware type. Since VM Classes created via the vSphere UI always have + // a default firmware value set (efi), this can cause VM boot failures for unsupported images. + if imageFirmware != "" { + // TODO: Use image firmware only when the class config spec has an empty firmware type. configSpec.Firmware = imageFirmware } diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/configspec_test.go b/pkg/vmprovider/providers/vsphere/virtualmachine/configspec_test.go index 5d213b46b..299ed4e71 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/configspec_test.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/configspec_test.go @@ -71,6 +71,7 @@ var _ = Describe("CreateConfigSpec", func() { }, }, }, + Firmware: "bios", } }) @@ -99,6 +100,17 @@ var _ = Describe("CreateConfigSpec", func() { Expect(ok).To(BeTrue()) }) + + When("image firmware is empty", func() { + BeforeEach(func() { + firmware = "" + }) + + It("config spec has the firmware from the class", func() { + Expect(configSpec.Firmware).ToNot(Equal(firmware)) + Expect(configSpec.Firmware).To(Equal(classConfigSpec.Firmware)) + }) + }) }) }) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go index 5ae236f88..b61a865b9 100644 --- a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go @@ -47,10 +47,14 @@ func CreateConfigSpec( Type: constants.ManagedByExtensionType, } - if val, ok := vmCtx.VM.Annotations[constants.FirmwareOverrideAnnotation]; ok { + if val, ok := vmCtx.VM.Annotations[constants.FirmwareOverrideAnnotation]; ok && (val == "efi" || val == "bios") { configSpec.Firmware = val - } else if configSpec.Firmware == "" && vmImageStatus != nil { - // Use firmware type from the image if ConfigSpec doesn't have it. + } else if vmImageStatus != nil && vmImageStatus.Firmware != "" { + // Use the image's firmware type if present. + // This is necessary until the vSphere UI can support creating VM Classes with + // an empty/nil firmware type. Since VM Classes created via the vSphere UI always has + // a non-empty firmware value set, this can cause VM boot failures. + // TODO: Use image firmware only when the class config spec has an empty firmware type. configSpec.Firmware = vmImageStatus.Firmware } diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go index 0dd01e72c..39625fead 100644 --- a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec_test.go @@ -13,6 +13,7 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" "github.com/vmware-tanzu/vm-operator/test/builder" ) @@ -21,6 +22,7 @@ var _ = Describe("CreateConfigSpec", func() { const vmName = "dummy-vm" var ( + vm *vmopv1.VirtualMachine vmCtx context.VirtualMachineContextA2 vmClassSpec *vmopv1.VirtualMachineClassSpec vmImageStatus *vmopv1.VirtualMachineImageStatus @@ -36,7 +38,7 @@ var _ = Describe("CreateConfigSpec", func() { vmImageStatus = &vmopv1.VirtualMachineImageStatus{Firmware: "efi"} minCPUFreq = 2500 - vm := builder.DummyVirtualMachineA2() + vm = builder.DummyVirtualMachineA2() vm.Name = vmName vmCtx = context.VirtualMachineContextA2{ Context: goctx.Background(), @@ -81,6 +83,7 @@ var _ = Describe("CreateConfigSpec", func() { }, }, }, + Firmware: "bios", } }) @@ -108,6 +111,37 @@ var _ = Describe("CreateConfigSpec", func() { _, ok := dSpec.Device.(*vimtypes.VirtualE1000) Expect(ok).To(BeTrue()) }) + + When("Image firmware is empty", func() { + BeforeEach(func() { + vmImageStatus = &vmopv1.VirtualMachineImageStatus{} + }) + + It("config spec has the firmware from the class", func() { + Expect(configSpec.Firmware).ToNot(Equal(vmImageStatus.Firmware)) + Expect(configSpec.Firmware).To(Equal(classConfigSpec.Firmware)) + }) + }) + + When("vm has a valid firmware override annotation", func() { + BeforeEach(func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = "efi" + }) + + It("config spec has overridden firmware annotation", func() { + Expect(configSpec.Firmware).To(Equal(vm.Annotations[constants.FirmwareOverrideAnnotation])) + }) + }) + + When("vm has an invalid firmware override annotation", func() { + BeforeEach(func() { + vm.Annotations[constants.FirmwareOverrideAnnotation] = "foo" + }) + + It("config spec doesn't have the invalid val", func() { + Expect(configSpec.Firmware).ToNot(Equal(vm.Annotations[constants.FirmwareOverrideAnnotation])) + }) + }) }) }) From 4b6afd4e35c559b3a8ceb47ce45aec3dac9fe56c Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Thu, 5 Oct 2023 13:01:10 -0500 Subject: [PATCH 22/54] Improve v1a2 VM Network validation checks First pass at better checks of the various Network fields during create. The existing table driven tests have reached their limit and are just too cumbersome to keep adding cases to. And it is annoying that the setup and assertion messages are not grouped together. Start new table driven tests for these new Network tests, and we should revisit the rest all the existing tests later once we've moved fully to v1a2. And I want to change we do the tests so that it is easier to share assertions between create and update. Correct the Gateway4 and Gateway6 docs: these should not have the network prefix, just the IP. --- api/v1alpha2/virtualmachine_network_types.go | 10 +- ...vmoperator.vmware.com_virtualmachines.yaml | 26 +- .../validation/virtualmachine_validator.go | 148 ++++++++-- .../virtualmachine_validator_unit_test.go | 266 ++++++++++++++++++ 4 files changed, 406 insertions(+), 44 deletions(-) diff --git a/api/v1alpha2/virtualmachine_network_types.go b/api/v1alpha2/virtualmachine_network_types.go index e8dfd5fc8..745054891 100644 --- a/api/v1alpha2/virtualmachine_network_types.go +++ b/api/v1alpha2/virtualmachine_network_types.go @@ -11,10 +11,10 @@ import ( // VirtualMachineNetworkRouteSpec defines a static route for a guest. type VirtualMachineNetworkRouteSpec struct { - // To is an IP4 address. + // To is an IP4 or IP6 address. To string `json:"to"` - // Via is an IP4 address. + // Via is an IP4 or IP6 address. Via string `json:"via"` // Metric is the weight/priority of the route. @@ -275,9 +275,6 @@ type VirtualMachineNetworkSpec struct { // Addresses field includes at least one IP4 address, then this field // is required. // - // Please note the IP address must include the network prefix length, ex. - // 192.168.0.1/24. - // // Please note this field is mutually exclusive with DHCP4. // // Please note if the Interfaces field is non-empty then this field is @@ -295,9 +292,6 @@ type VirtualMachineNetworkSpec struct { // Addresses field includes at least one IP4 address, then this field // is required. // - // Please note the IP address must include the network prefix length, ex. - // 2001:db8:101::1/64. - // // Please note this field is mutually exclusive with DHCP6. // // Please note if the Interfaces field is non-empty then this field is diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml index 06c3acfbd..f1822746a 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml @@ -1582,11 +1582,10 @@ spec: supports manual IP allocation. \n If the network connection supports manual IP allocation and the Addresses field includes at least one IP4 address, then this field is required. \n Please - note the IP address must include the network prefix length, - ex. 192.168.0.1/24. \n Please note this field is mutually exclusive - with DHCP4. \n Please note if the Interfaces field is non-empty - then this field is ignored and should be specified on the elements - in the Interfaces list." + note this field is mutually exclusive with DHCP4. \n Please + note if the Interfaces field is non-empty then this field is + ignored and should be specified on the elements in the Interfaces + list." type: string gateway6: description: "Gateway6 is the primary IP6 gateway for this VM. @@ -1594,11 +1593,10 @@ spec: supports manual IP allocation. \n If the network connection supports manual IP allocation and the Addresses field includes at least one IP4 address, then this field is required. \n Please - note the IP address must include the network prefix length, - ex. 2001:db8:101::1/64. \n Please note this field is mutually - exclusive with DHCP6. \n Please note if the Interfaces field - is non-empty then this field is ignored and should be specified - on the elements in the Interfaces list." + note this field is mutually exclusive with DHCP6. \n Please + note if the Interfaces field is non-empty then this field is + ignored and should be specified on the elements in the Interfaces + list." type: string hostName: description: "HostName is the value the guest uses as its host @@ -1730,10 +1728,10 @@ spec: format: int32 type: integer to: - description: To is an IP4 address. + description: To is an IP4 or IP6 address. type: string via: - description: Via is an IP4 address. + description: Via is an IP4 or IP6 address. type: string required: - metric @@ -1819,10 +1817,10 @@ spec: format: int32 type: integer to: - description: To is an IP4 address. + description: To is an IP4 or IP6 address. type: string via: - description: Via is an IP4 address. + description: Via is an IP4 or IP6 address. type: string required: - metric diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index ae13bf987..cc94d9d2e 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -314,46 +314,150 @@ func (v validator) validateNetwork(ctx *context.WebhookRequestContext, vm *vmopv networkSpec := &vm.Spec.Network networkPath := field.NewPath("spec", "network") - var hasIPv4Address, hasIPv6Address bool - for i, ipCIDR := range networkSpec.Addresses { + defaultNetworkInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ + Name: networkSpec.DeviceName, + Addresses: networkSpec.Addresses, + DHCP4: networkSpec.DHCP4, + DHCP6: networkSpec.DHCP6, + Gateway4: networkSpec.Gateway4, + Gateway6: networkSpec.Gateway6, + MTU: networkSpec.MTU, + Nameservers: networkSpec.Nameservers, + Routes: networkSpec.Routes, + SearchDomains: networkSpec.SearchDomains, + } + if networkSpec.Network != nil { + defaultNetworkInterface.Network = *networkSpec.Network + } + hasDefaultInterface := !equality.Semantic.DeepEqual(defaultNetworkInterface, vmopv1.VirtualMachineNetworkInterfaceSpec{}) + + if hasDefaultInterface { + if defaultNetworkInterface.Name == "" { + defaultNetworkInterface.Name = "eth0" + } + allErrs = append(allErrs, v.validateNetworkInterfaceSpec(networkPath, defaultNetworkInterface)...) + } + + if len(networkSpec.Interfaces) > 0 { + p := networkPath.Child("interfaces") + + if hasDefaultInterface { + // TODO: Better phrasing of this error message? + allErrs = append(allErrs, field.Invalid(p, nil, + "interfaces are mutually exclusive with deviceName,network,addresses,dhcp4,dhcp6,gateway4,"+ + "gateway6,mtu,nameservers,routes,searchDomains fields")) + } + + for i, interfaceSpec := range networkSpec.Interfaces { + allErrs = append(allErrs, v.validateNetworkInterfaceSpec(p.Index(i), interfaceSpec)...) + } + } + + return allErrs +} + +func (v validator) validateNetworkInterfaceSpec( + interfacePath *field.Path, + interfaceSpec vmopv1.VirtualMachineNetworkInterfaceSpec) field.ErrorList { + + var allErrs field.ErrorList + + // TODO: Ensure valid name once we finalize the naming convention for the network interface CRD. + + var ipv4Addrs, ipv6Addrs []string + for i, ipCIDR := range interfaceSpec.Addresses { ip, _, err := net.ParseCIDR(ipCIDR) if err != nil { - p := networkPath.Child("addresses").Index(i) + p := interfacePath.Child("addresses").Index(i) allErrs = append(allErrs, field.Invalid(p, ipCIDR, err.Error())) continue } - hasIPv4Address = hasIPv4Address || ip.To4() != nil - hasIPv6Address = hasIPv6Address || ip.To16() != nil + if ip.To4() != nil { + ipv4Addrs = append(ipv4Addrs, ipCIDR) + } else { + ipv6Addrs = append(ipv6Addrs, ipCIDR) + } + } + + if ipv4 := interfaceSpec.Gateway4; ipv4 != "" { + p := interfacePath.Child("gateway4") + + if len(ipv4Addrs) == 0 { + allErrs = append(allErrs, field.Invalid(p, ipv4, "gateway4 must have an IPv4 address in the addresses field")) + } + + if ip := net.ParseIP(ipv4); ip == nil || ip.To4() == nil { + allErrs = append(allErrs, field.Invalid(p, ipv4, "must be a valid IPv4 address")) + } } - if networkSpec.DHCP4 { - if hasIPv4Address { - p := networkPath.Child("dhcp4") - allErrs = append(allErrs, field.Invalid(p, networkSpec.Addresses, - "dhcp4 cannot be used with IP4 addresses in network.addresses field")) + if ipv6 := interfaceSpec.Gateway6; ipv6 != "" { + p := interfacePath.Child("gateway6") + + if len(ipv6Addrs) == 0 { + allErrs = append(allErrs, field.Invalid(p, ipv6, "gateway6 must have an IPv6 address in the addresses field")) } - if networkSpec.Gateway4 != "" { - p := networkPath.Child("gateway4") - allErrs = append(allErrs, field.Invalid(p, true, "gateway4 is mutually exclusive with dhcp4")) + if ip := net.ParseIP(ipv6); ip == nil || ip.To16() == nil || ip.To4() != nil { + allErrs = append(allErrs, field.Invalid(p, ipv6, "must be a valid IPv6 address")) } } - if networkSpec.DHCP6 { - if hasIPv6Address { - p := networkPath.Child("dhcp6") - allErrs = append(allErrs, field.Invalid(p, networkSpec.Addresses, - "dhcp6 cannot be used with IP6 addresses in network.addresses field")) + if interfaceSpec.DHCP4 { + if len(ipv4Addrs) > 0 { + p := interfacePath.Child("dhcp4") + allErrs = append(allErrs, field.Invalid(p, strings.Join(ipv4Addrs, ","), + "dhcp4 cannot be used with IPv4 addresses in addresses field")) } - if networkSpec.Gateway6 != "" { - p := networkPath.Child("gateway6") - allErrs = append(allErrs, field.Invalid(p, true, "gateway6 is mutually exclusive with dhcp6")) + if gw := interfaceSpec.Gateway4; gw != "" { + p := interfacePath.Child("gateway4") + allErrs = append(allErrs, field.Invalid(p, gw, "gateway4 is mutually exclusive with dhcp4")) + } + } + + if interfaceSpec.DHCP6 { + if len(ipv6Addrs) > 0 { + p := interfacePath.Child("dhcp6") + allErrs = append(allErrs, field.Invalid(p, strings.Join(ipv6Addrs, ","), + "dhcp6 cannot be used with IPv6 addresses in addresses field")) + } + + if gw := interfaceSpec.Gateway6; gw != "" { + p := interfacePath.Child("gateway6") + allErrs = append(allErrs, field.Invalid(p, gw, "gateway6 is mutually exclusive with dhcp6")) + } + } + + for i, n := range interfaceSpec.Nameservers { + if net.ParseIP(n) == nil { + allErrs = append(allErrs, + field.Invalid(interfacePath.Child("nameservers").Index(i), n, "must be an IPv4 or IPv6 address")) } } - // TODO: Much more + if len(interfaceSpec.Routes) > 0 { + p := interfacePath.Child("routes") + + for i, r := range interfaceSpec.Routes { + ip, _, err := net.ParseCIDR(r.To) + if err != nil { + allErrs = append(allErrs, field.Invalid(p.Index(i).Child("to"), r.To, err.Error())) + } + + viaIP := net.ParseIP(r.Via) + if viaIP == nil { + allErrs = append(allErrs, + field.Invalid(p.Index(i).Child("via"), r.Via, "must be an IPv4 or IPv6 address")) + } + + if (ip.To4() != nil) != (viaIP.To4() != nil) { + allErrs = append(allErrs, + field.Invalid(p.Index(i), "", "cannot mix IP address families")) + } + } + } return allErrs } diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go index 4bb3c797c..0b650047a 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -380,6 +381,271 @@ func unitTestsValidateCreate() { }, ", "), nil), Entry("should allow creating VM with admin-only annotations set by service user", createArgs{isServiceUser: true, adminOnlyAnnotations: true}, true, nil, nil), ) + + Context("Network", func() { + + type testParams struct { + setup func(ctx *unitValidatingWebhookContext) + validate func(response admission.Response) + expectAllowed bool + } + + doTest := func(args testParams) { + args.setup(ctx) + + var err error + ctx.WebhookRequestContext.Obj, err = builder.ToUnstructured(ctx.vm) + Expect(err).ToNot(HaveOccurred()) + + response := ctx.ValidateCreate(&ctx.WebhookRequestContext) + Expect(response.Allowed).To(Equal(args.expectAllowed)) + + if args.validate != nil { + args.validate(response) + } + } + + doValidateWithMsg := func(msgs ...string) func(admission.Response) { + return func(response admission.Response) { + reasons := strings.Split(string(response.Result.Reason), ", ") + for _, m := range msgs { + Expect(reasons).To(ContainElement(m)) + } + // This may be overly strict in some cases but catches missed assertions. + Expect(reasons).To(HaveLen(len(msgs))) + } + } + + DescribeTable("network create", doTest, + Entry("allow default", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{} + }, + expectAllowed: true, + }, + ), + + Entry("allow static", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + HostName: "my-vm", + DeviceName: "eth0", + Addresses: []string{ + "192.168.1.100/24", + "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", + }, + DHCP4: false, + DHCP6: false, + Gateway4: "192.168.1.1", + Gateway6: "2605:a601:a0ba:720:2ce6::1", + MTU: pointer.Int64(9000), + Nameservers: []string{ + "8.8.8.8", + "2001:4860:4860::8888", + }, + Routes: []vmopv1.VirtualMachineNetworkRouteSpec{ + { + To: "10.100.10.1/24", + Via: "10.10.1.1", + Metric: 42, + }, + { + To: "fbd6:93e7:bc11:18b2:514f:2b1d:637a:f695/48", + Via: "ef71:6ce2:3b91:8349:b2b2:f76c:86ae:915b", + }, + }, + SearchDomains: []string{"dev.local"}, + } + }, + expectAllowed: true, + }, + ), + + Entry("allow dhcp", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + HostName: "my-vm", + DeviceName: "eth0", + DHCP4: true, + DHCP6: true, + } + }, + expectAllowed: true, + }, + ), + + Entry("disallow mixing static and dhcp", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + HostName: "my-vm", + DeviceName: "eth0", + Addresses: []string{ + "192.168.1.100/24", + "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", + }, + DHCP4: true, + DHCP6: true, + Gateway4: "192.168.1.1", + Gateway6: "2605:a601:a0ba:720:2ce6::1", + } + }, + validate: doValidateWithMsg( + `spec.network.dhcp4: Invalid value: "192.168.1.100/24": dhcp4 cannot be used with IPv4 addresses in addresses field`, + `spec.network.gateway4: Invalid value: "192.168.1.1": gateway4 is mutually exclusive with dhcp4`, + `spec.network.dhcp6: Invalid value: "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48": dhcp6 cannot be used with IPv6 addresses in addresses field`, + `spec.network.gateway6: Invalid value: "2605:a601:a0ba:720:2ce6::1": gateway6 is mutually exclusive with dhcp6`, + ), + }, + ), + + Entry("validate addresses", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network.Addresses = []string{ + "1.1.", + "1.1.1.1", + "not-an-ip", + "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072", + } + }, + validate: doValidateWithMsg( + `spec.network.addresses[0]: Invalid value: "1.1.": invalid CIDR address: 1.1.`, + `spec.network.addresses[1]: Invalid value: "1.1.1.1": invalid CIDR address: 1.1.1.1`, + `spec.network.addresses[2]: Invalid value: "not-an-ip": invalid CIDR address: not-an-ip`, + `spec.network.addresses[3]: Invalid value: "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072": invalid CIDR address: 7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072`, + ), + }, + ), + + Entry("validate gateway4", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network.Gateway4 = "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072" + }, + validate: doValidateWithMsg( + `spec.network.gateway4: Invalid value: "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072": gateway4 must have an IPv4 address in the addresses field`, + `spec.network.gateway4: Invalid value: "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072": must be a valid IPv4 address`, + ), + }, + ), + + Entry("validate gateway6", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network.Gateway6 = "192.168.1.1" + }, + validate: doValidateWithMsg( + `spec.network.gateway6: Invalid value: "192.168.1.1": gateway6 must have an IPv6 address in the addresses field`, + `spec.network.gateway6: Invalid value: "192.168.1.1": must be a valid IPv6 address`, + ), + }, + ), + + Entry("validate nameservers", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network.Nameservers = []string{ + "not-an-ip", + "192.168.1.1/24", + } + }, + validate: doValidateWithMsg( + `spec.network.nameservers[0]: Invalid value: "not-an-ip": must be an IPv4 or IPv6 address`, + `spec.network.nameservers[1]: Invalid value: "192.168.1.1/24": must be an IPv4 or IPv6 address`, + ), + }, + ), + + Entry("validate routes", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network.Routes = []vmopv1.VirtualMachineNetworkRouteSpec{ + { + To: "10.100.10.1", + Via: "192.168.1", + }, + { + To: "2605:a601:a0ba:720:2ce6::/48", + Via: "2463:foobar", + }, + { + To: "192.168.1.1/24", + Via: "ef71:6ce2:3b91:8349:b2b2:f76c:86ae:915b", + }, + } + }, + validate: doValidateWithMsg( + `spec.network.routes[0].to: Invalid value: "10.100.10.1": invalid CIDR address: 10.100.10.1`, + `spec.network.routes[0].via: Invalid value: "192.168.1": must be an IPv4 or IPv6 address`, + `spec.network.routes[1].via: Invalid value: "2463:foobar": must be an IPv4 or IPv6 address`, + `spec.network.routes[2]: Invalid value: "": cannot mix IP address families`, + ), + }, + ), + + Entry("validate interfaces", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + HostName: "my-vm", + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Addresses: []string{ + "192.168.1.100/24", + "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", + }, + Gateway4: "192.168.1.1", + Gateway6: "2605:a601:a0ba:720:2ce6::1", + MTU: pointer.Int64(9000), + Nameservers: []string{ + "8.8.8.8", + "2001:4860:4860::8888", + }, + Routes: []vmopv1.VirtualMachineNetworkRouteSpec{ + { + To: "10.100.10.1/24", + Via: "10.10.1.1", + Metric: 42, + }, + { + To: "fbd6:93e7:bc11:18b2:514f:2b1d:637a:f695/48", + Via: "ef71:6ce2:3b91:8349:b2b2:f76c:86ae:915b", + }, + }, + SearchDomains: []string{"dev.local"}, + }, + }, + } + }, + expectAllowed: true, + }, + ), + + Entry("disallow interfaces with default interface", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + HostName: "my-vm", + Addresses: []string{"192.168.1.10/24"}, + Gateway4: "192.168.1.1", + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + DHCP4: true, + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.network.interfaces: Invalid value: "null": interfaces are mutually exclusive with deviceName,network,addresses,dhcp4,dhcp6,gateway4,gateway6,mtu,nameservers,routes,searchDomains fields`, + ), + }, + ), + ) + }) } func unitTestsValidateUpdate() { From c78ce24c9667b4bede469b2f197651f845f399a8 Mon Sep 17 00:00:00 2001 From: Arunesh Pandey Date: Wed, 11 Oct 2023 12:13:36 -0700 Subject: [PATCH 23/54] Add vulncheck local target and GH action Golang recently released Govulncheck https://go.dev/blog/govulncheck which allows us to check our repo for vulnerabilities found in Golang standard libraries. This change also adds a Github action for the same when any pull request is opened or merged on main. --- .github/workflows/ci.yml | 24 +++++++++++++++++++++- Dockerfile | 2 +- Makefile | 12 ++++++++++- go.mod | 10 +++++----- go.sum | 28 ++++++++++++++++++-------- hack/tools/Makefile | 6 ++++++ hack/tools/go.mod | 19 +++++++++--------- hack/tools/go.sum | 43 ++++++++++++++++++++++++++-------------- hack/tools/tools.go | 1 + 9 files changed, 105 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86cd93469..adbceb1e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: ci env: - GO_VERSION: 1.20.0 + GO_VERSION: 1.21.3 on: pull_request: @@ -86,6 +86,28 @@ jobs: - name: Lint Go run: make lint-go-full + vulncheck-go: + needs: + - verify-go-modules + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + cache-dependency-path: '**/go.sum' + - name: Setup the cache for govulncheck + uses: actions/cache@v3 + with: + key: govulncheck-${{ runner.os }}-go${{ env.GO_VERSION }}-${{ hashFiles('go.sum', 'hack/tools/go.sum') }} + path: | + hack/tools/bin/govulncheck + - name: Vulncheck Go + run: make vulncheck-go + build-manager: needs: - verify-go-modules diff --git a/Dockerfile b/Dockerfile index f1cf4274b..dab396ba8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Go version used to build the binaries. -ARG GO_VERSION=1.20 +ARG GO_VERSION=1.21.3 ## Docker image used to build the binaries. FROM golang:${GO_VERSION} as builder diff --git a/Makefile b/Makefile index c513aa456..bb101cd2b 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,7 @@ KUBE_APISERVER := $(TOOLS_BIN_DIR)/kube-apiserver KUBEBUILDER := $(TOOLS_BIN_DIR)/kubebuilder KUBECTL := $(TOOLS_BIN_DIR)/kubectl ETCD := $(TOOLS_BIN_DIR)/etcd +GOVULNCHECK := $(TOOLS_BIN_DIR)/govulncheck # Allow overriding manifest generation destination directory MANIFEST_ROOT ?= config @@ -193,7 +194,7 @@ web-console-validator: prereqs generate lint-go web-console-validator-only ## Bu TOOLING_BINARIES := $(CRD_REF_DOCS) $(CONTROLLER_GEN) $(CONVERSION_GEN) \ $(GOLANGCI_LINT) $(KUSTOMIZE) \ $(KUBE_APISERVER) $(KUBEBUILDER) $(KUBECTL) $(ETCD) \ - $(GINKGO) $(GOCOVMERGE) $(GOCOV) $(GOCOV_XML) + $(GINKGO) $(GOCOVMERGE) $(GOCOV) $(GOCOV_XML) $(GOVULNCHECK) tools: $(TOOLING_BINARIES) ## Build tooling binaries .PHONY: $(TOOLING_BINARIES) $(TOOLING_BINARIES): @@ -552,6 +553,15 @@ docker-remove: ## Remove the docker image fi +## -------------------------------------- +## Vulnerability Checks +## -------------------------------------- + +.PHONY: vulncheck-go +vulncheck-go: $(GOVULNCHECK) + $(GOVULNCHECK) ./... + + ## -------------------------------------- ## Clean and verify ## -------------------------------------- diff --git a/go.mod b/go.mod index a37ee17d3..864d7648c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/vmware-tanzu/vm-operator -go 1.20 +go 1.21 replace ( github.com/envoyproxy/go-control-plane => github.com/envoyproxy/go-control-plane v0.9.4 @@ -25,8 +25,8 @@ require ( github.com/vmware/govmomi v0.31.0 // per the following dependabot alerts: // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/24 - golang.org/x/net v0.13.0 // indirect - golang.org/x/text v0.11.0 + golang.org/x/net v0.17.0 // indirect + golang.org/x/text v0.13.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/grpc v1.54.0 gopkg.in/yaml.v2 v2.4.0 @@ -72,8 +72,8 @@ require ( github.com/prometheus/procfs v0.10.1 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/term v0.10.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect diff --git a/go.sum b/go.sum index 610b42d4d..4d5dd7bbb 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,7 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -158,6 +159,7 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -220,6 +222,7 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -296,6 +299,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -357,6 +361,7 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -408,6 +413,7 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -449,6 +455,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -472,14 +479,18 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -554,8 +565,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= -golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -617,13 +628,13 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -632,8 +643,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -685,6 +696,7 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/hack/tools/Makefile b/hack/tools/Makefile index 3d5f6c1ef..b51ca9edf 100644 --- a/hack/tools/Makefile +++ b/hack/tools/Makefile @@ -40,6 +40,7 @@ KUBECTL := $(BIN_DIR)/kubectl KIND := $(BIN_DIR)/kind GOCOV := $(BIN_DIR)/gocov GOCOV_XML := $(BIN_DIR)/gocov-xml +GOVULNCHECK := $(BIN_DIR)/govulncheck ## -------------------------------------- ## Help @@ -116,6 +117,11 @@ kubectl: $(KUBECTL) ## Install kubectl $(ETCD) $(KUBE_APISERVER) $(KUBECTL): k8s-envtest ## Install envtest related binaries @cp -f "$(ENVTEST_DOWNLOAD_DIR)/$(@F)" $(BIN_DIR) +.PHONY: $(GOVULNCHECK) +govulncheck: $(GOVULNCHECK) ## Install govulncheck +$(GOVULNCHECK): go.mod + go build -tags=vmop_tools -o $@ golang.org/x/vuln/cmd/govulncheck + .PHONY: $(KIND) kind: $(KIND) ## Install kind $(KIND): diff --git a/hack/tools/go.mod b/hack/tools/go.mod index 41441019d..a6a6d6938 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -1,6 +1,6 @@ module github.com/vmware-tanzu/vm-operator/hack/tools -go 1.18 +go 1.21 require ( github.com/AlekSi/gocov-xml v1.1.0 @@ -9,6 +9,7 @@ require ( github.com/golangci/golangci-lint v1.51.2 github.com/onsi/ginkgo v1.16.5 github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad + golang.org/x/vuln v1.0.1 k8s.io/code-generator v0.26.1 sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20220825200008-d242fe21e646 sigs.k8s.io/controller-tools v0.10.0 @@ -195,15 +196,15 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect - golang.org/x/crypto v0.5.0 // indirect + golang.org/x/crypto v0.12.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -222,7 +223,7 @@ require ( mvdan.cc/gofumpt v0.4.0 // indirect mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect - mvdan.cc/unparam v0.0.0-20221223090309-7455f1af531d // indirect + mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/hack/tools/go.sum b/hack/tools/go.sum index c60b3fa39..8ca8999be 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -147,6 +147,7 @@ github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4 github.com/firefart/nonamedreturns v1.0.4 h1:abzI1p7mAEPYuR4A+VLKn4eNDOycjYo2phmY9sfv40Y= github.com/firefart/nonamedreturns v1.0.4/go.mod h1:TDhe/tjI1BXo48CmYbUduTV7BdIga8MAO/xbKdcVsGI= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -200,6 +201,7 @@ github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlN github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= github.com/go-toolsmith/pkgload v1.0.2-0.20220101231613-e814995d17c5 h1:eD9POs68PHkwrx7hAB78z1cb6PfGq/jyWn3wJywsH1o= +github.com/go-toolsmith/pkgload v1.0.2-0.20220101231613-e814995d17c5/go.mod h1:3NAwwmD4uY/yggRxoEjk/S00MIV3A+H7rrE3i87eYxM= github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= @@ -270,6 +272,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU= +github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -299,6 +303,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -322,6 +327,7 @@ github.com/gostaticanalysis/nilerr v0.1.1 h1:ThE+hJP0fEp4zWLkWHWcRyI2Od0p7DlgYG3 github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW0HU0GPE3+5PWN4A= github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoISdUv3PPQgHY= +github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= @@ -380,6 +386,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -472,9 +479,11 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.8.0 h1:pAM+oBNPrpXRs+E/8spkeGx9QgekbRVyr74EUvRVOUI= +github.com/onsi/ginkgo/v2 v2.8.0/go.mod h1:6JsQiECmxCa3V5st74AL/AmsV482EDdVrGaVW6z3oYU= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= @@ -530,6 +539,7 @@ github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4l github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.3.0 h1:q15RT/pd6UggBXVBuLps8BXRvl5GPBcwVA7BJHMLuTw= github.com/ryancurrah/gomodguard v1.3.0/go.mod h1:ggBxb3luypPEzqVtq33ee7YSN35V28XeGnid8dnni50= @@ -662,8 +672,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -708,8 +718,8 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -755,8 +765,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -780,8 +790,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -849,8 +860,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -868,8 +879,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -953,8 +964,10 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/vuln v1.0.1 h1:KUas02EjQK5LTuIx1OylBQdKKZ9jeugs+HiqO5HormU= +golang.org/x/vuln v1.0.1/go.mod h1:bb2hMwln/tqxg32BNY4CcxHWtHXuYa3SbIBmtsyjxtM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1116,8 +1129,8 @@ mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wp mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= -mvdan.cc/unparam v0.0.0-20221223090309-7455f1af531d h1:3rvTIIM22r9pvXk+q3swxUQAQOxksVMGK7sml4nG57w= -mvdan.cc/unparam v0.0.0-20221223090309-7455f1af531d/go.mod h1:IeHQjmn6TOD+e4Z3RFiZMMsLVL+A96Nvptar8Fj71is= +mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8 h1:VuJo4Mt0EVPychre4fNlDWDuE5AjXtPJpRUWqZDQhaI= +mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8/go.mod h1:Oh/d7dEtzsNHGOq1Cdv8aMm3KdKhVvPbRQcM8WFpBR8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/hack/tools/tools.go b/hack/tools/tools.go index 017f36800..a941ed720 100644 --- a/hack/tools/tools.go +++ b/hack/tools/tools.go @@ -27,6 +27,7 @@ import ( _ "github.com/golangci/golangci-lint/cmd/golangci-lint" _ "github.com/onsi/ginkgo/ginkgo" _ "github.com/wadey/gocovmerge" + _ "golang.org/x/vuln/cmd/govulncheck" _ "k8s.io/code-generator" _ "sigs.k8s.io/controller-runtime/tools/setup-envtest" _ "sigs.k8s.io/controller-tools/cmd/controller-gen" From 77d0ce96d2b405d22643dafa67b4d346ab9635b5 Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Thu, 12 Oct 2023 12:34:05 -0700 Subject: [PATCH 24/54] Add v1a2 webconsolerequest validation webhooks (#242) --- config/webhook/manifests.yaml | 21 ++ test/builder/utila2.go | 13 ++ .../validation/webconsolerequest_validator.go | 189 ++++++++++++++++ .../webconsolerequest_validator_intg_test.go | 119 ++++++++++ .../webconsolerequest_validator_suite_test.go | 29 +++ .../webconsolerequest_validator_unit_test.go | 205 ++++++++++++++++++ .../v1alpha2/webhooks.go | 20 ++ .../webhooks.go | 7 + 8 files changed, 603 insertions(+) create mode 100644 webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator.go create mode 100644 webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_intg_test.go create mode 100644 webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_suite_test.go create mode 100644 webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_unit_test.go create mode 100644 webhooks/virtualmachinewebconsolerequest/v1alpha2/webhooks.go diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 8b124ee10..d848c383b 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -306,3 +306,24 @@ webhooks: resources: - webconsolerequests sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinewebconsolerequest + failurePolicy: Fail + name: default.validating.virtualmachinewebconsolerequest.v1alpha2.vmoperator.vmware.com + rules: + - apiGroups: + - vmoperator.vmware.com + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - virtualmachinewebconsolerequests + sideEffects: None diff --git a/test/builder/utila2.go b/test/builder/utila2.go index 902587367..141d93cf5 100644 --- a/test/builder/utila2.go +++ b/test/builder/utila2.go @@ -301,3 +301,16 @@ func DummyClusterVirtualMachineImageA2(imageName string) *vmopv1.ClusterVirtualM }, } } + +func DummyVirtualMachineWebConsoleRequest(namespace, wcrName, vmName, pubKey string) *vmopv1.VirtualMachineWebConsoleRequest { + return &vmopv1.VirtualMachineWebConsoleRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: wcrName, + Namespace: namespace, + }, + Spec: vmopv1.VirtualMachineWebConsoleRequestSpec{ + Name: vmName, + PublicKey: pubKey, + }, + } +} diff --git a/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator.go b/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator.go new file mode 100644 index 000000000..67b75c92f --- /dev/null +++ b/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator.go @@ -0,0 +1,189 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package validation + +import ( + "crypto/x509" + "encoding/pem" + "net/http" + "reflect" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/api/validation" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + webconsolerequest "github.com/vmware-tanzu/vm-operator/controllers/virtualmachinewebconsolerequest/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/builder" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/webhooks/common" +) + +const ( + webHookName = "default" +) + +// +kubebuilder:webhook:verbs=create;update,path=/default-validate-vmoperator-vmware-com-v1alpha2-virtualmachinewebconsolerequest,mutating=false,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachinewebconsolerequests,versions=v1alpha2,name=default.validating.virtualmachinewebconsolerequest.v1alpha2.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinewebconsolerequests,verbs=get;list +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinewebconsolerequests/status,verbs=get + +// AddToManager adds the webhook to the provided manager. +func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + hook, err := builder.NewValidatingWebhook(ctx, mgr, webHookName, NewValidator(mgr.GetClient())) + if err != nil { + return errors.Wrapf(err, "failed to create virtualmachinewebconsolerequest validation webhook") + } + mgr.GetWebhookServer().Register(hook.Path, hook) + return nil +} + +// NewValidator returns the package's Validator. +func NewValidator(_ client.Client) builder.Validator { + return validator{ + converter: runtime.DefaultUnstructuredConverter, + } +} + +type validator struct { + converter runtime.UnstructuredConverter +} + +func (v validator) For() schema.GroupVersionKind { + return vmopv1.SchemeGroupVersion.WithKind(reflect.TypeOf(vmopv1.VirtualMachineWebConsoleRequest{}).Name()) +} + +func (v validator) ValidateCreate(ctx *context.WebhookRequestContext) admission.Response { + + wcr, err := v.webConsoleRequestFromUnstructured(ctx.Obj) + if err != nil { + return webhook.Errored(http.StatusBadRequest, err) + } + + var fieldErrs field.ErrorList + fieldErrs = append(fieldErrs, v.validateMetadata(ctx, wcr)...) + fieldErrs = append(fieldErrs, v.validateSpec(ctx, wcr)...) + + validationErrs := make([]string, 0, len(fieldErrs)) + for _, fieldErr := range fieldErrs { + validationErrs = append(validationErrs, fieldErr.Error()) + } + + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) +} + +func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Response { + return admission.Allowed("") +} + +func (v validator) ValidateUpdate(ctx *context.WebhookRequestContext) admission.Response { + wcr, err := v.webConsoleRequestFromUnstructured(ctx.Obj) + if err != nil { + return webhook.Errored(http.StatusBadRequest, err) + } + + oldwcr, err := v.webConsoleRequestFromUnstructured(ctx.OldObj) + if err != nil { + return webhook.Errored(http.StatusBadRequest, err) + } + + var fieldErrs field.ErrorList + fieldErrs = append(fieldErrs, v.validateImmutableFields(wcr, oldwcr)...) + fieldErrs = append(fieldErrs, v.validateUUIDLabel(wcr, oldwcr)...) + + validationErrs := make([]string, 0, len(fieldErrs)) + for _, fieldErr := range fieldErrs { + validationErrs = append(validationErrs, fieldErr.Error()) + } + return common.BuildValidationResponse(ctx, nil, validationErrs, nil) +} + +func (v validator) validateMetadata(ctx *context.WebhookRequestContext, wcr *vmopv1.VirtualMachineWebConsoleRequest) field.ErrorList { + var fieldErrs field.ErrorList + return fieldErrs +} + +func (v validator) validateSpec(ctx *context.WebhookRequestContext, wcr *vmopv1.VirtualMachineWebConsoleRequest) field.ErrorList { + var fieldErrs field.ErrorList + specPath := field.NewPath("spec") + + fieldErrs = append(fieldErrs, v.validateVirtualMachineName(specPath.Child("Name"), wcr)...) + fieldErrs = append(fieldErrs, v.validatePublicKey(specPath.Child("publicKey"), wcr.Spec.PublicKey)...) + + return fieldErrs +} + +func (v validator) validateVirtualMachineName(path *field.Path, wcr *vmopv1.VirtualMachineWebConsoleRequest) field.ErrorList { + var allErrs field.ErrorList + + if wcr.Spec.Name == "" { + allErrs = append(allErrs, field.Required(path, "")) + return allErrs + } + + // Not checking existence of wcr.Spec.Name because the webhooks are meant for internal consistency + // checking. Also, the kubectl-vsphere plugin validates the existence of the VM at that level as well. + + return allErrs +} + +func (v validator) validatePublicKey(path *field.Path, publicKey string) field.ErrorList { + var allErrs field.ErrorList + + if publicKey == "" { + allErrs = append(allErrs, field.Required(path, "")) + return allErrs + } + + block, _ := pem.Decode([]byte(publicKey)) + if block == nil || block.Type != "PUBLIC KEY" { + allErrs = append(allErrs, field.Invalid(path, "", "invalid public key format")) + return allErrs + } + _, err := x509.ParsePKCS1PublicKey(block.Bytes) + if err != nil { + allErrs = append(allErrs, field.Invalid(path, "", "invalid public key")) + } + + return allErrs +} + +func (v validator) validateImmutableFields(wcr, oldwcr *vmopv1.VirtualMachineWebConsoleRequest) field.ErrorList { + var allErrs field.ErrorList + specPath := field.NewPath("spec") + + allErrs = append(allErrs, validation.ValidateImmutableField(wcr.Spec.Name, oldwcr.Spec.Name, specPath.Child("Name"))...) + allErrs = append(allErrs, validation.ValidateImmutableField(wcr.Spec.PublicKey, oldwcr.Spec.PublicKey, specPath.Child("publicKey"))...) + + return allErrs +} + +// webConsoleRequestFromUnstructured returns the wcr from the unstructured object. +func (v validator) webConsoleRequestFromUnstructured(obj runtime.Unstructured) (*vmopv1.VirtualMachineWebConsoleRequest, error) { + wcr := &vmopv1.VirtualMachineWebConsoleRequest{} + if err := v.converter.FromUnstructured(obj.UnstructuredContent(), wcr); err != nil { + return nil, err + } + return wcr, nil +} + +func (v validator) validateUUIDLabel(wcr, oldwcr *vmopv1.VirtualMachineWebConsoleRequest) field.ErrorList { + var allErrs field.ErrorList + + oldUUIDLabelVal := oldwcr.Labels[webconsolerequest.UUIDLabelKey] + if oldUUIDLabelVal == "" { + return allErrs + } + + newUUIDLabelVal := wcr.Labels[webconsolerequest.UUIDLabelKey] + labelsPath := field.NewPath("metadata", "labels") + allErrs = append(allErrs, validation.ValidateImmutableField(newUUIDLabelVal, oldUUIDLabelVal, labelsPath.Key(webconsolerequest.UUIDLabelKey))...) + + return allErrs +} diff --git a/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_intg_test.go b/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_intg_test.go new file mode 100644 index 000000000..df44c463f --- /dev/null +++ b/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_intg_test.go @@ -0,0 +1,119 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package validation_test + +import ( + "crypto/rsa" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func intgTests() { + Describe("Invoking Create", intgTestsValidateCreate) + Describe("Invoking Update", intgTestsValidateUpdate) + Describe("Invoking Delete", intgTestsValidateDelete) +} + +type intgValidatingWebhookContext struct { + builder.IntegrationTestContext + wcr *vmopv1.VirtualMachineWebConsoleRequest + privateKey *rsa.PrivateKey +} + +func newIntgValidatingWebhookContext() *intgValidatingWebhookContext { + privateKey, publicKeyPem := builder.WebConsoleRequestKeyPair() + + ctx := &intgValidatingWebhookContext{ + IntegrationTestContext: *suite.NewIntegrationTestContext(), + } + + ctx.wcr = builder.DummyVirtualMachineWebConsoleRequest(ctx.Namespace, "some-name", "some-vm-name", publicKeyPem) + ctx.privateKey = privateKey + return ctx +} + +func intgTestsValidateCreate() { + var ( + err error + ctx *intgValidatingWebhookContext + ) + BeforeEach(func() { + ctx = newIntgValidatingWebhookContext() + }) + AfterEach(func() { + err = nil + ctx = nil + }) + + When("create is performed", func() { + BeforeEach(func() { + err = ctx.Client.Create(ctx, ctx.wcr) + }) + It("should allow the request", func() { + Expect(err).ToNot(HaveOccurred()) + }) + }) +} + +func intgTestsValidateUpdate() { + var ( + err error + ctx *intgValidatingWebhookContext + ) + + BeforeEach(func() { + ctx = newIntgValidatingWebhookContext() + err = ctx.Client.Create(ctx, ctx.wcr) + Expect(err).ToNot(HaveOccurred()) + }) + JustBeforeEach(func() { + err = ctx.Client.Update(suite, ctx.wcr) + }) + AfterEach(func() { + + err = nil + ctx = nil + }) + + When("update is performed with changed vm name", func() { + BeforeEach(func() { + ctx.wcr.Spec.Name = "alternate-vm-name" + }) + It("should deny the request", func() { + Expect(err).To(HaveOccurred()) + }) + }) +} + +func intgTestsValidateDelete() { + var ( + err error + ctx *intgValidatingWebhookContext + ) + + BeforeEach(func() { + ctx = newIntgValidatingWebhookContext() + err = ctx.Client.Create(ctx, ctx.wcr) + Expect(err).ToNot(HaveOccurred()) + }) + JustBeforeEach(func() { + err = ctx.Client.Delete(suite, ctx.wcr) + }) + AfterEach(func() { + + err = nil + ctx = nil + }) + + When("delete is performed", func() { + It("should allow the request", func() { + Expect(ctx.Namespace).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + }) + }) +} diff --git a/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_suite_test.go b/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_suite_test.go new file mode 100644 index 000000000..803f7826d --- /dev/null +++ b/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_suite_test.go @@ -0,0 +1,29 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package validation_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + + "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/test/builder" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation" +) + +// suite is used for unit and integration testing this webhook. +var suite = builder.NewTestSuiteForValidatingWebhookwithFSS( + validation.AddToManager, + validation.NewValidator, + "default.validating.virtualmachinewebconsolerequest.v1alpha2.vmoperator.vmware.com", + map[string]bool{lib.VMServiceV1Alpha2FSS: true}) + +func TestWebhook(t *testing.T) { + suite.Register(t, "Validation webhook suite", intgTests, unitTests) +} + +var _ = BeforeSuite(suite.BeforeSuite) + +var _ = AfterSuite(suite.AfterSuite) diff --git a/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_unit_test.go b/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_unit_test.go new file mode 100644 index 000000000..68e703fb1 --- /dev/null +++ b/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation/webconsolerequest_validator_unit_test.go @@ -0,0 +1,205 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package validation_test + +import ( + "crypto/rsa" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/controllers/virtualmachinewebconsolerequest/v1alpha2" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func unitTests() { + Describe("Invoking ValidateCreate", unitTestsValidateCreate) + Describe("Invoking ValidateUpdate", unitTestsValidateUpdate) + Describe("Invoking ValidateDelete", unitTestsValidateDelete) +} + +type unitValidatingWebhookContext struct { + builder.UnitTestContextForValidatingWebhook + wcr *vmopv1.VirtualMachineWebConsoleRequest + oldWcr *vmopv1.VirtualMachineWebConsoleRequest + privateKey *rsa.PrivateKey +} + +func newUnitTestContextForValidatingWebhook(isUpdate bool) *unitValidatingWebhookContext { + privateKey, publicKeyPem := builder.WebConsoleRequestKeyPair() + + wcr := builder.DummyVirtualMachineWebConsoleRequest("some-namespace", "some-name", "some-vm-name", publicKeyPem) + wcr.Labels = map[string]string{ + v1alpha2.UUIDLabelKey: "some-uuid", + } + obj, err := builder.ToUnstructured(wcr) + Expect(err).ToNot(HaveOccurred()) + + var oldWcr *vmopv1.VirtualMachineWebConsoleRequest + var oldObj *unstructured.Unstructured + + if isUpdate { + oldWcr = wcr.DeepCopy() + oldObj, err = builder.ToUnstructured(oldWcr) + Expect(err).ToNot(HaveOccurred()) + } + + return &unitValidatingWebhookContext{ + UnitTestContextForValidatingWebhook: *suite.NewUnitTestContextForValidatingWebhook(obj, oldObj), + wcr: wcr, + oldWcr: oldWcr, + privateKey: privateKey, + } +} + +func unitTestsValidateCreate() { + var ( + ctx *unitValidatingWebhookContext + ) + + type createArgs struct { + emptyVirtualMachineName bool + emptyPublicKey bool + invalidPublicKey bool + } + + validateCreate := func(args createArgs, expectedAllowed bool, expectedReason string, expectedErr error) { + var err error + + if args.emptyVirtualMachineName { + ctx.wcr.Spec.Name = "" + } + if args.emptyPublicKey { + ctx.wcr.Spec.PublicKey = "" + } + if args.invalidPublicKey { + ctx.wcr.Spec.PublicKey = "invalid-public-key" + } + + ctx.WebhookRequestContext.Obj, err = builder.ToUnstructured(ctx.wcr) + Expect(err).ToNot(HaveOccurred()) + + response := ctx.ValidateCreate(&ctx.WebhookRequestContext) + Expect(response.Allowed).To(Equal(expectedAllowed)) + if expectedReason != "" { + Expect(string(response.Result.Reason)).To(ContainSubstring(expectedReason)) + } + if expectedErr != nil { + Expect(response.Result.Message).To(Equal(expectedErr.Error())) + } + } + + BeforeEach(func() { + ctx = newUnitTestContextForValidatingWebhook(false) + }) + AfterEach(func() { + ctx = nil + }) + + DescribeTable("create table", validateCreate, + Entry("should allow valid", createArgs{}, true, nil, nil), + Entry("should deny empty virtualmachinename", createArgs{emptyVirtualMachineName: true}, false, "spec.Name: Required value", nil), + Entry("should deny empty publickey", createArgs{emptyPublicKey: true}, false, "spec.publicKey: Required value", nil), + Entry("should deny invalid publickey", createArgs{invalidPublicKey: true}, false, "spec.publicKey: Invalid value: \"\": invalid public key format", nil), + ) +} + +func unitTestsValidateUpdate() { + var ( + ctx *unitValidatingWebhookContext + response admission.Response + ) + + type updateArgs struct { + updateVirtualMachineName bool + updatePublicKey bool + updateUUIDLabel bool + } + + validateUpdate := func(args updateArgs, expectedAllowed bool, expectedReason string, expectedErr error) { + var err error + + if args.updateVirtualMachineName { + ctx.wcr.Spec.Name = "new-vm-name" + } + + if args.updatePublicKey { + ctx.wcr.Spec.PublicKey = "new-public-key" + } + + if args.updateUUIDLabel { + ctx.wcr.Labels[v1alpha2.UUIDLabelKey] = "new-uuid" + } + + ctx.WebhookRequestContext.Obj, err = builder.ToUnstructured((ctx.wcr)) + Expect(err).ToNot(HaveOccurred()) + + response := ctx.ValidateUpdate(&ctx.WebhookRequestContext) + Expect(response.Allowed).To(Equal(expectedAllowed)) + if expectedReason != "" { + Expect(string(response.Result.Reason)).To(Equal(expectedReason)) + } + if expectedErr != nil { + Expect(response.Result.Message).To(Equal(expectedErr.Error())) + } + } + + BeforeEach(func() { + ctx = newUnitTestContextForValidatingWebhook(true) + }) + AfterEach(func() { + ctx = nil + }) + + DescribeTable("update table", validateUpdate, + Entry("should allow", updateArgs{}, true, nil, nil), + Entry("should deny Virtualmachine Name change", updateArgs{updateVirtualMachineName: true}, false, "spec.Name: Invalid value: \"new-vm-name\": field is immutable", nil), + Entry("should deny PublicKey change", updateArgs{updatePublicKey: true}, false, "spec.publicKey: Invalid value: \"new-public-key\": field is immutable", nil), + Entry("should deny UUID label change", updateArgs{updateUUIDLabel: true}, false, "metadata.labels[vmoperator.vmware.com/webconsolerequest-uuid]: Invalid value: \"new-uuid\": field is immutable", nil), + ) + + When("the update is performed while object deletion", func() { + JustBeforeEach(func() { + t := metav1.Now() + ctx.WebhookRequestContext.Obj.SetDeletionTimestamp(&t) + response = ctx.ValidateUpdate(&ctx.WebhookRequestContext) + }) + + It("should allow the request", func() { + Expect(response.Allowed).To(BeTrue()) + Expect(response.Result).ToNot(BeNil()) + }) + }) +} + +func unitTestsValidateDelete() { + var ( + ctx *unitValidatingWebhookContext + response admission.Response + ) + + BeforeEach(func() { + ctx = newUnitTestContextForValidatingWebhook(false) + }) + AfterEach(func() { + ctx = nil + }) + + When("the delete is performed", func() { + JustBeforeEach(func() { + response = ctx.ValidateDelete(&ctx.WebhookRequestContext) + }) + + It("should allow the request", func() { + Expect(response.Allowed).To(BeTrue()) + Expect(response.Result).ToNot(BeNil()) + }) + }) +} diff --git a/webhooks/virtualmachinewebconsolerequest/v1alpha2/webhooks.go b/webhooks/virtualmachinewebconsolerequest/v1alpha2/webhooks.go new file mode 100644 index 000000000..40af30ca5 --- /dev/null +++ b/webhooks/virtualmachinewebconsolerequest/v1alpha2/webhooks.go @@ -0,0 +1,20 @@ +// Copyright (c) 2022-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha2 + +import ( + "github.com/pkg/errors" + + ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinewebconsolerequest/v1alpha2/validation" +) + +func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if err := validation.AddToManager(ctx, mgr); err != nil { + return errors.Wrap(err, "failed to initialize validation webhook") + } + return nil +} diff --git a/webhooks/virtualmachinewebconsolerequest/webhooks.go b/webhooks/virtualmachinewebconsolerequest/webhooks.go index 8d1a9a17e..ef4c5b3f7 100644 --- a/webhooks/virtualmachinewebconsolerequest/webhooks.go +++ b/webhooks/virtualmachinewebconsolerequest/webhooks.go @@ -7,9 +7,16 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinewebconsolerequest/v1alpha1" + "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachinewebconsolerequest/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if lib.IsVMServiceV1Alpha2FSSEnabled() { + if err := v1alpha2.AddToManager(ctx, mgr); err != nil { + return err + } + } return v1alpha1.AddToManager(ctx, mgr) } From ee7ca92cebe878d3ade26e56c3db6acf9b7d9c38 Mon Sep 17 00:00:00 2001 From: akutz Date: Thu, 12 Oct 2023 14:56:03 -0500 Subject: [PATCH 25/54] Platform-aware tooling This patch makes all of the hack/tools tooling binaries platform aware by placing them in hack/tools/bin/GOHOSTOS_GHOSTARCH, also ensuring the binaries are built using the platform, not using GOOS/GOARCH. Additionally, the tools targets are no longer marked PHONY, nor are the tools order-only pre-reqs for other targets. Because most tooling is managed via Go modules, the go.mod file is a pre-req for building tools. This means that tools will be built once per platform and only rebuilt if go.mod changes. --- Makefile | 34 +++---- hack/ensure-kustomize.sh | 49 ---------- hack/tools/Makefile | 126 +++++++++++++------------- hack/tools/go.mod | 73 +++++++++------ hack/tools/go.sum | 189 ++++++++++++++++++++++----------------- hack/tools/tools.go | 22 ++--- 6 files changed, 242 insertions(+), 251 deletions(-) delete mode 100755 hack/ensure-kustomize.sh diff --git a/Makefile b/Makefile index bb101cd2b..1c797ec28 100644 --- a/Makefile +++ b/Makefile @@ -19,10 +19,15 @@ else # (,$(strip $(shell command -v go 2>/dev/null || true))) # Active module mode, as we use go modules to manage dependencies. export GO111MODULE := on +# Get the information about the platform on which the tools are built/run. +GOHOSTOS := $(shell go env GOHOSTOS) +GOHOSTARCH := $(shell go env GOHOSTARCH) +GOHOSTOSARCH := $(GOHOSTOS)_$(GOHOSTARCH) + # Default the GOOS and GOARCH values to be the same as the platform on which # this Makefile is being executed. -export GOOS ?= $(shell go env GOHOSTOS) -export GOARCH ?= $(shell go env GOHOSTARCH) +export GOOS ?= $(GOHOSTOS) +export GOARCH ?= $(GOHOSTARCH) # The directory in which this Makefile is located. Please note this will not # behave correctly if the path to any Makefile in the list contains any spaces. @@ -48,7 +53,7 @@ endif # ifeq (,$(strip $(shell command -v go 2>/dev/null || true))) # Directories BIN_DIR := bin TOOLS_DIR := hack/tools -TOOLS_BIN_DIR := $(TOOLS_DIR)/bin +TOOLS_BIN_DIR := $(TOOLS_DIR)/bin/$(GOHOSTOSARCH) UPGRADE_DIR := upgrade export PATH := $(abspath $(BIN_DIR)):$(abspath $(TOOLS_BIN_DIR)):$(PATH) export KUBEBUILDER_ASSETS := $(abspath $(TOOLS_BIN_DIR)) @@ -141,18 +146,18 @@ test-nocover: ## Run Tests (without code coverage) hack/test-unit.sh .PHONY: test -test: | $(GOCOVMERGE) +test: $(GOCOVMERGE) test: ## Run tests @rm -f $(COVERAGE_FILE) hack/test-unit.sh $(COVERAGE_FILE) .PHONY: test-integration -test-integration: | $(ETCD) $(KUBE_APISERVER) +test-integration: $(ETCD) $(KUBE_APISERVER) test-integration: ## Run integration tests KUBECONFIG=$(KUBECONFIG) hack/test-integration.sh $(INT_COV_FILE) .PHONY: coverage -coverage-merge: | $(GOCOVMERGE) $(GOCOV) $(GOCOV_XML) +coverage-merge: $(GOCOVMERGE) $(GOCOV) $(GOCOV_XML) coverage-merge: ## Merge the coverage from unit and integration tests $(GOCOVMERGE) $(COVERAGE_FILE) $(INT_COV_FILE) >$(FULL_COV_FILE) gocov convert "$(FULL_COV_FILE)" | gocov-xml >"$(FULL_COV_FILE:.out=.xml)" @@ -196,7 +201,6 @@ TOOLING_BINARIES := $(CRD_REF_DOCS) $(CONTROLLER_GEN) $(CONVERSION_GEN) \ $(KUBE_APISERVER) $(KUBEBUILDER) $(KUBECTL) $(ETCD) \ $(GINKGO) $(GOCOVMERGE) $(GOCOV) $(GOCOV_XML) $(GOVULNCHECK) tools: $(TOOLING_BINARIES) ## Build tooling binaries -.PHONY: $(TOOLING_BINARIES) $(TOOLING_BINARIES): make -C $(TOOLS_DIR) $(@F) @@ -212,7 +216,7 @@ lint: ## Run all the lint targets GOLANGCI_LINT_FLAGS ?= --fast=true .PHONY: lint-go -lint-go: | $(GOLANGCI_LINT) +lint-go: $(GOLANGCI_LINT) lint-go: ## Lint codebase $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_FLAGS) @@ -262,14 +266,14 @@ generate: ## Generate code # $(MAKE) generate-api-docs .PHONY: generate-go -generate-go: | $(CONTROLLER_GEN) +generate-go: $(CONTROLLER_GEN) generate-go: ## Generate deepcopy $(CONTROLLER_GEN) \ paths=github.com/vmware-tanzu/vm-operator/api/... \ object:headerFile=./hack/boilerplate/boilerplate.generatego.txt .PHONY: generate-manifests -generate-manifests: | $(CONTROLLER_GEN) +generate-manifests: $(CONTROLLER_GEN) generate-manifests: ## Generate manifests e.g. CRD, RBAC etc. $(CONTROLLER_GEN) \ paths=github.com/vmware-tanzu/vm-operator/api/... \ @@ -288,7 +292,7 @@ generate-manifests: ## Generate manifests e.g. CRD, RBAC etc. rbac:roleName=manager-role .PHONY: generate-external-manifests -generate-external-manifests: | $(CONTROLLER_GEN) +generate-external-manifests: $(CONTROLLER_GEN) generate-external-manifests: ## Generate manifests for the external types for testing API_MOD_DIR=$(shell go mod download -json $(IMG_REGISTRY_OP_API_SLUG) | grep '"Dir":' | awk '{print $$2}' | tr -d '",') && \ $(CONTROLLER_GEN) \ @@ -312,7 +316,7 @@ ifneq (,$(ROOT_DIR_IN_GOPATH)) # statement ensures that there is not an order-only dependency on CONVERSION_GEN # if it already exists. ifeq (,$(strip $(wildcard $(CONVERSION_GEN)))) -generate-go-conversions: | $(CONVERSION_GEN) +generate-go-conversions: $(CONVERSION_GEN) endif generate-go-conversions: @@ -412,7 +416,7 @@ generate-go-conversions: endif .PHONY: generate-api-docs -generate-api-docs: | $(CRD_REF_DOCS) +generate-api-docs: $(CRD_REF_DOCS) generate-api-docs: ## Generate API documentation $(CRD_REF_DOCS) \ --renderer=markdown \ @@ -444,13 +448,13 @@ kustomize-x: .PHONY: kustomize-local kustomize-local: CONFIG_TYPE=local kustomize-local: YAML_OUT=$(LOCAL_YAML) -kustomize-local: prereqs generate-manifests | $(KUSTOMIZE) +kustomize-local: prereqs generate-manifests $(KUSTOMIZE) kustomize-local: kustomize-x ## Kustomize for local cluster .PHONY: kustomize-local-vcsim kustomize-local-vcsim: CONFIG_TYPE=local-vcsim kustomize-local-vcsim: YAML_OUT=$(LOCAL_YAML) -kustomize-local-vcsim: prereqs generate-manifests | $(KUSTOMIZE) +kustomize-local-vcsim: prereqs generate-manifests $(KUSTOMIZE) kustomize-local-vcsim: kustomize-x ## Kustomize for local-vcsim cluster diff --git a/hack/ensure-kustomize.sh b/hack/ensure-kustomize.sh deleted file mode 100755 index 89c77dbaa..000000000 --- a/hack/ensure-kustomize.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -BIN_ROOT="${KUBE_ROOT}/hack/tools/bin" - -kustomize_version=4.4.1 - -goos=$(go env GOOS) -goarch=$(go env GOARCH) - -if [ "$goos" != "linux" ] && [ "$goos" != "darwin" ]; then - echo "OS '$goos' not supported. Aborting." >&2 - exit 1 -fi - -# Ensure the kustomize tool exists and is a viable version, or installs it -verify_kustomize_version() { - if ! [ -x "$(command -v "${BIN_ROOT}/kustomize")" ]; then - echo "fetching kustomize@${kustomize_version}" - if ! [ -d "${BIN_ROOT}" ]; then - mkdir -p "${BIN_ROOT}" - fi - archive_name="kustomize-v${kustomize_version}.tar.gz" - curl -sLo "${BIN_ROOT}/${archive_name}" "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${kustomize_version}/kustomize_v${kustomize_version}_${goos}_${goarch}.tar.gz" - tar -zvxf "${BIN_ROOT}/${archive_name}" -C "${BIN_ROOT}/" - chmod +x "${BIN_ROOT}/kustomize" - rm "${BIN_ROOT}/${archive_name}" - fi -} - -verify_kustomize_version diff --git a/hack/tools/Makefile b/hack/tools/Makefile index b51ca9edf..d672cdb56 100644 --- a/hack/tools/Makefile +++ b/hack/tools/Makefile @@ -10,37 +10,38 @@ SHELL := /usr/bin/env bash export GO111MODULE := on # Versions. -KUBEBUILDER_VERSION=3.3.0 K8S_VERSION=1.24.1 -KIND_VERSION=0.11.1 -HOST_OS=$(shell go env GOOS) -HOST_ARCH=$(shell go env GOARCH) +# Get the information about the platform on which the tools are built/run. +GOHOSTOS := $(shell go env GOHOSTOS) +GOHOSTARCH := $(shell go env GOHOSTARCH) +GOHOSTOSARCH := $(GOHOSTOS)_$(GOHOSTARCH) + +# Default the GOOS and GOARCH values to be the same as the platform on which +# this Makefile is being executed. +export GOOS := $(GOHOSTOS) +export GOARCH := $(GOHOSTARCH) # Directories. BIN_DIR := bin -# The SETUP_ENVTEST download directory. setup-envtest will install binaries here, -# but it is easiest if we copy them to our BIN_DIR. -ENVTEST_DOWNLOAD_DIR = $(shell $(SETUP_ENVTEST) use -p path $(K8S_VERSION)) - # Binaries. -CRD_REF_DOCS := $(BIN_DIR)/crd-ref-docs -CONTROLLER_GEN := $(BIN_DIR)/controller-gen -CONVERSION_GEN := $(BIN_DIR)/conversion-gen -SETUP_ENVTEST := $(BIN_DIR)/setup-envtest -GOLANGCI_LINT := $(BIN_DIR)/golangci-lint -KUSTOMIZE := $(BIN_DIR)/kustomize -GINKGO := $(BIN_DIR)/ginkgo -GOCOVMERGE := $(BIN_DIR)/gocovmerge -KUBEBUILDER := $(BIN_DIR)/kubebuilder -ETCD := $(BIN_DIR)/etcd -KUBE_APISERVER := $(BIN_DIR)/kube-apiserver -KUBECTL := $(BIN_DIR)/kubectl -KIND := $(BIN_DIR)/kind -GOCOV := $(BIN_DIR)/gocov -GOCOV_XML := $(BIN_DIR)/gocov-xml -GOVULNCHECK := $(BIN_DIR)/govulncheck +CRD_REF_DOCS := $(BIN_DIR)/$(GOHOSTOSARCH)/crd-ref-docs +CONTROLLER_GEN := $(BIN_DIR)/$(GOHOSTOSARCH)/controller-gen +CONVERSION_GEN := $(BIN_DIR)/$(GOHOSTOSARCH)/conversion-gen +SETUP_ENVTEST := $(BIN_DIR)/$(GOHOSTOSARCH)/setup-envtest +GOLANGCI_LINT := $(BIN_DIR)/$(GOHOSTOSARCH)/golangci-lint +KUSTOMIZE := $(BIN_DIR)/$(GOHOSTOSARCH)/kustomize +GINKGO := $(BIN_DIR)/$(GOHOSTOSARCH)/ginkgo +GOCOVMERGE := $(BIN_DIR)/$(GOHOSTOSARCH)/gocovmerge +KUBEBUILDER := $(BIN_DIR)/$(GOHOSTOSARCH)/kubebuilder +ETCD := $(BIN_DIR)/$(GOHOSTOSARCH)/etcd +KUBE_APISERVER := $(BIN_DIR)/$(GOHOSTOSARCH)/kube-apiserver +KUBECTL := $(BIN_DIR)/$(GOHOSTOSARCH)/kubectl +KIND := $(BIN_DIR)/$(GOHOSTOSARCH)/kind +GOCOV := $(BIN_DIR)/$(GOHOSTOSARCH)/gocov +GOCOV_XML := $(BIN_DIR)/$(GOHOSTOSARCH)/gocov-xml +GOVULNCHECK := $(BIN_DIR)/$(GOHOSTOSARCH)/govulncheck ## -------------------------------------- ## Help @@ -53,81 +54,82 @@ help: ## Display this help ## Binaries ## -------------------------------------- -.PHONY: $(CRD_REF_DOCS) crd-ref-docs: $(CRD_REF_DOCS) ## Install crd-ref-docs -$(CRD_REF_DOCS): - go build -tags=vmop_tools -o $@ github.com/elastic/crd-ref-docs +$(CRD_REF_DOCS): go.mod + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) github.com/elastic/crd-ref-docs -.PHONY: $(CONTROLLER_GEN) controller-gen: $(CONTROLLER_GEN) ## Install controller-gen $(CONTROLLER_GEN): go.mod - go build -tags=vmop_tools -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) sigs.k8s.io/controller-tools/cmd/controller-gen -.PHONY: $(CONVERSION_GEN) conversion-gen: $(CONVERSION_GEN) ## Install conversion-gen $(CONVERSION_GEN): go.mod - go build -tags=vmop_tools -o $@ k8s.io/code-generator/cmd/conversion-gen + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) k8s.io/code-generator/cmd/conversion-gen -.PHONY:$(SETUP_ENVTEST) setup-envtest: $(SETUP_ENVTEST) ## Install setup-envtest $(SETUP_ENVTEST): go.mod - go build -tags=vmop_tools -o $@ sigs.k8s.io/controller-runtime/tools/setup-envtest + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) sigs.k8s.io/controller-runtime/tools/setup-envtest -.PHONY: $(GOLANGCI_LINT) golangci-lint: $(GOLANGCI_LINT) ## Install golangci-lint -$(GOLANGCI_LINT): - go build -tags=vmop_tools -o $@ github.com/golangci/golangci-lint/cmd/golangci-lint +$(GOLANGCI_LINT): go.mod + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) github.com/golangci/golangci-lint/cmd/golangci-lint -.PHONY: $(KUSTOMIZE) kustomize: $(KUSTOMIZE) ## Install kustomize -$(KUSTOMIZE): - ../ensure-kustomize.sh +$(KUSTOMIZE): go.mod + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) sigs.k8s.io/kustomize/kustomize/v5 -.PHONY: $(GINKGO) ginkgo: $(GINKGO) ## Install ginkgo $(GINKGO): go.mod - go build -tags=vmop_tools -o $@ github.com/onsi/ginkgo/ginkgo + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) github.com/onsi/ginkgo/ginkgo -.PHONY: $(GOCOVMERGE) gocovmerge: $(GOCOVMERGE) ## Install gocovmerge $(GOCOVMERGE): go.mod - go build -tags=vmop_tools -o $@ github.com/wadey/gocovmerge + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) github.com/wadey/gocovmerge -.PHONY: $(GOCOV) gocov: $(GOCOV) ## Install gocov $(GOCOV): go.mod - go build -tags=vmop_tools -o $@ github.com/axw/gocov/gocov + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) github.com/axw/gocov/gocov -.PHONY: $(GOCOV_XML) gocov-xml: $(GOCOV_XML) ## Install gocov-xml $(GOCOV_XML): go.mod - go build -tags=vmop_tools -o $@ github.com/AlekSi/gocov-xml + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) github.com/AlekSi/gocov-xml -.PHONY: $(KUBEBUILDER) kubebuilder: $(KUBEBUILDER) ## Install kubebuilder - curl -sL https://github.com/kubernetes-sigs/kubebuilder/releases/download/v$(KUBEBUILDER_VERSION)/kubebuilder_$(HOST_OS)_$(HOST_ARCH) --output $(KUBEBUILDER) && chmod a+x $(KUBEBUILDER) +$(KUBEBUILDER): go.mod + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) sigs.k8s.io/kubebuilder/v3/cmd + +kind: $(KIND) ## Install kind +$(KIND): go.mod + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) sigs.k8s.io/kind + +govulncheck: $(GOVULNCHECK) ## Install govulncheck +$(GOVULNCHECK): go.mod + GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) \ + go build -tags=vmop_tools -o $(@) golang.org/x/vuln/cmd/govulncheck .PHONY: k8s-envtest k8s-envtest: $(SETUP_ENVTEST) ## Download envtest binaries - $(SETUP_ENVTEST) use --os $(HOST_OS) --arch $(HOST_ARCH) $(K8S_VERSION) + $(SETUP_ENVTEST) use --os $(GOHOSTOS) --arch $(GOHOSTARCH) $(K8S_VERSION) etcd: $(ETCD) ## Install etcd kube-apiserver: $(KUBE_APISERVER) ## Install kube-apiserver kubectl: $(KUBECTL) ## Install kubectl $(ETCD) $(KUBE_APISERVER) $(KUBECTL): k8s-envtest ## Install envtest related binaries - @cp -f "$(ENVTEST_DOWNLOAD_DIR)/$(@F)" $(BIN_DIR) - -.PHONY: $(GOVULNCHECK) -govulncheck: $(GOVULNCHECK) ## Install govulncheck -$(GOVULNCHECK): go.mod - go build -tags=vmop_tools -o $@ golang.org/x/vuln/cmd/govulncheck - -.PHONY: $(KIND) -kind: $(KIND) ## Install kind -$(KIND): @mkdir -p $(@D) - curl -sL https://github.com/kubernetes-sigs/kind/releases/download/v$(KIND_VERSION)/kind-$(HOST_OS)-$(HOST_ARCH) -o $(@) && \ - chmod a+x $(@) + @cp -f "$$($(SETUP_ENVTEST) use -p path $(K8S_VERSION))/$(@F)" $(@) + ## -------------------------------------- ## Generate diff --git a/hack/tools/go.mod b/hack/tools/go.mod index a6a6d6938..1a2a25962 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -13,6 +13,9 @@ require ( k8s.io/code-generator v0.26.1 sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20220825200008-d242fe21e646 sigs.k8s.io/controller-tools v0.10.0 + sigs.k8s.io/kind v0.20.0 + sigs.k8s.io/kubebuilder/v3 v3.12.0 + sigs.k8s.io/kustomize/kustomize/v5 v5.1.1 ) require ( @@ -28,6 +31,7 @@ require ( github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/OpenPeeDeeP/depguard v1.1.1 // indirect + github.com/alessio/shellescape v1.4.1 // indirect github.com/alexkohler/prealloc v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect github.com/ashanbrown/forbidigo v1.4.0 // indirect @@ -49,18 +53,21 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/esimonov/ifshort v1.0.4 // indirect github.com/ettle/strcase v0.1.1 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.14.1 // indirect github.com/fatih/structtag v1.2.0 // indirect github.com/firefart/nonamedreturns v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect github.com/go-critic/go-critic v0.6.7 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/zapr v1.2.0 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect github.com/go-toolsmith/astcopy v1.0.3 // indirect github.com/go-toolsmith/astequal v1.1.0 // indirect @@ -69,12 +76,12 @@ require ( github.com/go-toolsmith/strparse v1.1.0 // indirect github.com/go-toolsmith/typep v1.1.0 // indirect github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect - github.com/gobuffalo/flect v0.3.0 // indirect + github.com/gobuffalo/flect v1.0.2 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-yaml v1.1.5 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe // indirect @@ -84,9 +91,11 @@ require ( github.com/golangci/misspell v0.4.0 // indirect github.com/golangci/revgrep v0.0.0-20220804021717-745bb2f7c2e6 // indirect github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gordonklaus/ineffassign v0.0.0-20230107090616-13ace0543b28 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect @@ -100,7 +109,7 @@ require ( github.com/hexops/gotextdiff v1.0.3 // indirect github.com/huandu/xstrings v1.2.1 // indirect github.com/imdario/mergo v0.3.6 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jgautheron/goconst v1.5.1 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect @@ -119,14 +128,14 @@ require ( github.com/leonklingele/grouper v1.1.1 // indirect github.com/lufeee/execinquery v1.2.1 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/maratori/testableexamples v1.0.0 // indirect github.com/maratori/testpackage v1.1.0 // indirect github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mbilski/exhaustivestruct v1.2.0 // indirect github.com/mgechev/revive v1.2.5 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect @@ -135,6 +144,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/moricho/tparallel v0.2.1 // indirect github.com/nakabonne/nestif v0.3.1 // indirect github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect @@ -148,10 +158,10 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polyfloyd/go-errorlint v1.1.0 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/client_golang v1.12.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/quasilyte/go-ruleguard v0.3.19 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect @@ -163,15 +173,15 @@ require ( github.com/sashamelentyev/usestdlibvars v1.23.0 // indirect github.com/securego/gosec/v2 v2.15.0 // indirect github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/sivchari/containedctx v1.0.2 // indirect github.com/sivchari/nosnakecase v1.7.0 // indirect github.com/sivchari/tenv v1.7.1 // indirect github.com/sonatard/noctx v0.0.1 // indirect github.com/sourcegraph/go-diff v0.7.0 // indirect - github.com/spf13/afero v1.8.2 // indirect + github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.6.1 // indirect + github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.12.0 // indirect @@ -190,9 +200,11 @@ require ( github.com/ultraware/funlen v0.0.3 // indirect github.com/ultraware/whitespace v0.0.5 // indirect github.com/uudashr/gocognit v1.0.6 // indirect + github.com/xlab/treeprint v1.2.0 // indirect github.com/yagipy/maintidx v1.0.0 // indirect github.com/yeya24/promlinter v0.2.0 // indirect gitlab.com/bosi/decorder v0.2.3 // indirect + go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect @@ -203,28 +215,31 @@ require ( golang.org/x/net v0.14.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect - golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect - google.golang.org/protobuf v1.28.1 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.4.2 // indirect - k8s.io/api v0.26.1 // indirect - k8s.io/apiextensions-apiserver v0.26.1 // indirect - k8s.io/apimachinery v0.26.1 // indirect + k8s.io/api v0.25.0 // indirect + k8s.io/apiextensions-apiserver v0.25.0 // indirect + k8s.io/apimachinery v0.25.0 // indirect k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect + k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961 // indirect + k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect mvdan.cc/gofumpt v0.4.0 // indirect mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect mvdan.cc/unparam v0.0.0-20230312165513-e84e2d14e3b8 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kustomize/api v0.14.0 // indirect + sigs.k8s.io/kustomize/cmd/config v0.11.3 // indirect + sigs.k8s.io/kustomize/kyaml v0.14.3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 8ca8999be..2da03ccf1 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -49,6 +49,7 @@ github.com/Antonboom/errname v0.1.7/go.mod h1:g0ONh16msHIPgJSGsecu1G/dcF2hlYR/0S github.com/Antonboom/nilnil v0.1.1 h1:PHhrh5ANKFWRBh7TdYmyyq2gyT2lotnvFvvFbylF81Q= github.com/Antonboom/nilnil v0.1.1/go.mod h1:L1jBqoWM7AOeTD+tSquifKSesRHs4ZdaxvZR+xdJEaI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -69,6 +70,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pOcUuw= github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= @@ -113,6 +116,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo= @@ -124,7 +128,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingaikin/go-header v0.4.3 h1:tEaZKAlqql6SKCY++utLmkPLd6K8IBM20Ha7UVm+mtU= github.com/denis-tingaikin/go-header v0.4.3/go.mod h1:0wOCWuN71D5qIgE2nz9KrKmuYBAC2Mra5RassOIQ2/c= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elastic/crd-ref-docs v0.0.9-0.20220728100728-3a11386f88f1 h1:7G0Et3YwZgR0vsrXWh5kfHzy/zUtfueE2rXtXS/V5As= github.com/elastic/crd-ref-docs v0.0.9-0.20220728100728-3a11386f88f1/go.mod h1:Jd1XDGgrvHzd/+qCf4SwBnOnLl4RaFRjind0ORnHovo= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= @@ -139,6 +142,10 @@ github.com/esimonov/ifshort v1.0.4 h1:6SID4yGWfRae/M7hkVDVVyppy8q/v9OuxNdmjLQStB github.com/esimonov/ifshort v1.0.4/go.mod h1:Pe8zjlRrJ80+q2CxHLfEOfTwxCZ4O+MuhcHcfgNWTk0= github.com/ettle/strcase v0.1.1 h1:htFueZyVeE1XNnMEfbqp5r67qAN/4r6ya1ysq8Q+Zcw= github.com/ettle/strcase v0.1.1/go.mod h1:hzDLsPC7/lwKyBOywSHEP89nt2pDgdy+No1NBA9o9VY= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= @@ -150,44 +157,43 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/go-critic/go-critic v0.6.7 h1:1evPrElnLQ2LZtJfmNDzlieDhjnq36SLgNzisx06oPM= github.com/go-critic/go-critic v0.6.7/go.mod h1:fYZUijFdcnxgx6wPjQA2QEjIRaNCT0gO8bhexy6/QmE= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= github.com/go-toolsmith/astcopy v1.0.3 h1:r0bgSRlMOAgO+BdQnVAcpMSMkrQCnV6ZJmIkrJgcJj0= @@ -209,8 +215,8 @@ github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUN github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U= github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= -github.com/gobuffalo/flect v0.3.0 h1:erfPWM+K1rFNIQeRPdeEXxo8yFr/PO17lhRnS8FUrtk= -github.com/gobuffalo/flect v0.3.0/go.mod h1:5pf3aGnsvqvCj50AVni7mJJF8ICxGZ8HomberC3pXLE= +github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= +github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-yaml v1.1.5 h1:b8mLuSkAaBDntdh7UUL2aU/PXZ7dPfsrNNG13VhvKGs= @@ -246,8 +252,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= @@ -270,8 +277,8 @@ github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSW github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU= github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -288,8 +295,9 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -303,8 +311,14 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -348,8 +362,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jgautheron/goconst v1.5.1 h1:HxVbL1MhydKs8R8n/HE5NPvzfaYmQJA3o879lE4+WcM= github.com/jgautheron/goconst v1.5.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs= @@ -385,6 +401,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -409,10 +426,8 @@ github.com/lufeee/execinquery v1.2.1 h1:hf0Ems4SHcUGBxpGN7Jz78z1ppVkP/837ZlETPCE github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maratori/testableexamples v1.0.0 h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s93SLMxb2vI= github.com/maratori/testableexamples v1.0.0/go.mod h1:4rhjL1n20TUTT4vdh3RDqSizKLyXp7K2u6HgraZCGzE= github.com/maratori/testpackage v1.1.0 h1:GJY4wlzQhuBusMF1oahQCBtUV/AQ/k69IZ68vxaac2Q= @@ -426,14 +441,14 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mbilski/exhaustivestruct v1.2.0 h1:wCBmUnSYufAHO6J4AVWY6ff+oxWxsVFrwgOdMUQePUo= github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc= github.com/mgechev/revive v1.2.5 h1:UF9AR8pOAuwNmhXj2odp4mxv9Nx2qUIwVz8ZsU+Mbec= @@ -453,6 +468,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/moricho/tparallel v0.2.1 h1:95FytivzT6rYzdJLdtfn6m1bfFJylOJK41+lgv/EHf4= github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -461,7 +478,6 @@ github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81 github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA= github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/exhaustive v0.9.5 h1:TzssWan6orBiLYVqewCG8faud9qlFntJE30ACpzmGME= github.com/nishanths/exhaustive v0.9.5/go.mod h1:IbwrGdVMizvDcIxPYGVdQn5BqWJaOwpCvg4RGb8r/TA= @@ -478,18 +494,19 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.8.0 h1:pAM+oBNPrpXRs+E/8spkeGx9QgekbRVyr74EUvRVOUI= -github.com/onsi/ginkgo/v2 v2.8.0/go.mod h1:6JsQiECmxCa3V5st74AL/AmsV482EDdVrGaVW6z3oYU= +github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= +github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q= -github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= @@ -507,28 +524,24 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/quasilyte/go-ruleguard v0.3.19 h1:tfMnabXle/HzOb5Xe9CUZYWXKfkS1KwRmZyPmD9nVcc= github.com/quasilyte/go-ruleguard v0.3.19/go.mod h1:lHSn69Scl48I7Gt9cX3VrbsZYvYiBYszZOZW4A+oTEw= github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= @@ -538,8 +551,8 @@ github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:r github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.3.0 h1:q15RT/pd6UggBXVBuLps8BXRvl5GPBcwVA7BJHMLuTw= github.com/ryancurrah/gomodguard v1.3.0/go.mod h1:ggBxb3luypPEzqVtq33ee7YSN35V28XeGnid8dnni50= @@ -553,6 +566,8 @@ github.com/sashamelentyev/usestdlibvars v1.23.0 h1:01h+/2Kd+NblNItNeux0veSL5cBF1 github.com/sashamelentyev/usestdlibvars v1.23.0/go.mod h1:YPwr/Y1LATzHI93CqoPUN/2BzGQ/6N/cl/KwgR0B/aU= github.com/securego/gosec/v2 v2.15.0 h1:v4Ym7FF58/jlykYmmhZ7mTm7FQvN/setNm++0fgIAtw= github.com/securego/gosec/v2 v2.15.0/go.mod h1:VOjTrZOkUtSDt2QLSJmQBMWnvwiQPEjg0l+5juIqGk8= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= @@ -560,8 +575,8 @@ github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOms github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sivchari/containedctx v1.0.2 h1:0hLQKpgC53OVF1VT7CeoFHk9YKstur1XOgfYIc1yrHI= github.com/sivchari/containedctx v1.0.2/go.mod h1:PwZOeqm4/DLoJOqMSIJs3aKqXRX4YO+uXww087KZ7Bw= github.com/sivchari/nosnakecase v1.7.0 h1:7QkpWIRMe8x25gckkFd2A5Pi6Ymo0qgr4JrhGt95do8= @@ -572,12 +587,13 @@ github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY= github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI= github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= -github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= -github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -588,7 +604,6 @@ github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YE github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stbenjam/no-sprintf-host-port v0.1.1 h1:tYugd/yrm1O0dV+ThCbaKZh195Dfm07ysF0U6JQXczc= github.com/stbenjam/no-sprintf-host-port v0.1.1/go.mod h1:TLhvtIvONRzdmkFiio4O8LHsN9N74I+PhRquPsxpL0I= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -633,6 +648,8 @@ github.com/uudashr/gocognit v1.0.6 h1:2Cgi6MweCsdB6kpcVQp7EW4U23iBFQWfTXiWlyp842 github.com/uudashr/gocognit v1.0.6/go.mod h1:nAIUuVBnYU7pcninia3BHOvQkpQCeO76Uscky5BOwcY= github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad h1:W0LEBv82YCGEtcmPA3uNZBI33/qF//HAAs3MawDjRa0= github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= github.com/yeya24/promlinter v0.2.0 h1:xFKDQ82orCU5jQujdaD8stOHiv8UN68BSdn2a8u8Y3o= @@ -652,6 +669,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= @@ -670,7 +689,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= @@ -758,8 +777,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= @@ -777,7 +795,6 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -809,6 +826,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -846,16 +864,16 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -879,8 +897,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -972,8 +990,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1030,7 +1048,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1065,14 +1082,15 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= @@ -1093,7 +1111,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1106,12 +1123,12 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.4.2 h1:6qXr+R5w+ktL5UkwEbPp+fEvfyoMPche6GkOpGHZcLc= honnef.co/go/tools v0.4.2/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= -k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= -k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= -k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= -k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= -k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= -k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= +k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= +k8s.io/apiextensions-apiserver v0.25.0 h1:CJ9zlyXAbq0FIW8CD7HHyozCMBpDSiH7EdrSTCZcZFY= +k8s.io/apiextensions-apiserver v0.25.0/go.mod h1:3pAjZiN4zw7R8aZC5gR0y3/vCkGlAjCazcg1me8iB/E= +k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= +k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= k8s.io/code-generator v0.26.1 h1:dusFDsnNSKlMFYhzIM0jAO1OlnTN5WYwQQ+Ai12IIlo= k8s.io/code-generator v0.26.1/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZmeI3I= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08= @@ -1119,10 +1136,10 @@ k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961 h1:pqRVJGQJz6oeZby8qmPKXYIBjyrcv7EHCe/33UkZMYA= +k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961/go.mod h1:l8HTwL5fqnlns4jOveW1L75eo7R9KFHxiE0bsPGy428= +k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU= +k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= @@ -1138,8 +1155,20 @@ sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20220825200008-d242fe2 sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20220825200008-d242fe21e646/go.mod h1:nLkMD2WB4Jcix1qfVuJeOF4j5y/VfyeOIlTxG5Wj9co= sigs.k8s.io/controller-tools v0.10.0 h1:0L5DTDTFB67jm9DkfrONgTGmfc/zYow0ZaHyppizU2U= sigs.k8s.io/controller-tools v0.10.0/go.mod h1:uvr0EW6IsprfB0jpQq6evtKy+hHyHCXNfdWI5ONPx94= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kind v0.20.0 h1:f0sc3v9mQbGnjBUaqSFST1dwIuiikKVGgoTwpoP33a8= +sigs.k8s.io/kind v0.20.0/go.mod h1:aBlbxg08cauDgZ612shr017/rZwqd7AS563FvpWKPVs= +sigs.k8s.io/kubebuilder/v3 v3.12.0 h1:POh46v+T2wPGzAzcIE/eKlwZr17nn+R49elJg2Wz2qU= +sigs.k8s.io/kubebuilder/v3 v3.12.0/go.mod h1:ZJZ6jpjhh0skfehrYl7b8X/SCozhfLAU7IYV8ZN/f3s= +sigs.k8s.io/kustomize/api v0.14.0 h1:6+QLmXXA8X4eDM7ejeaNUyruA1DDB3PVIjbpVhDOJRA= +sigs.k8s.io/kustomize/api v0.14.0/go.mod h1:vmOXlC8BcmcUJQjiceUbcyQ75JBP6eg8sgoyzc+eLpQ= +sigs.k8s.io/kustomize/cmd/config v0.11.3 h1:QLukJoe/0sjhUrtylmBS1MXhvkdLtbpHJvAClXDra54= +sigs.k8s.io/kustomize/cmd/config v0.11.3/go.mod h1:ENTZ8Ds12gewUpdxF5PJq/9qPVQFd5VPvMIL11wrBIU= +sigs.k8s.io/kustomize/kustomize/v5 v5.1.1 h1:iq+1k9LaQupKcbUVLX8yvE62W6u0B5bXtyCmF5YUcH8= +sigs.k8s.io/kustomize/kustomize/v5 v5.1.1/go.mod h1:7kno0pHkt7k3Vg4/0IjpMxx1bzCi08gziU2CTa6UuvM= +sigs.k8s.io/kustomize/kyaml v0.14.3 h1:WpabVAKZe2YEp/irTSHwD6bfjwZnTtSDewd2BVJGMZs= +sigs.k8s.io/kustomize/kyaml v0.14.3/go.mod h1:npvh9epWysfQ689Rtt/U+dpOJDTBn8kUnF1O6VzvmZA= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/hack/tools/tools.go b/hack/tools/tools.go index a941ed720..d5e323f2f 100644 --- a/hack/tools/tools.go +++ b/hack/tools/tools.go @@ -1,23 +1,10 @@ //go:build vmop_tools // +build vmop_tools -/* -Copyright 2019 The Kubernetes Authors. +// Copyright (c) 2019-2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This package imports things required by build scripts, to force `go mod` to see them as dependencies +// Package tools manages the version of tooling used to build this project. package tools import ( @@ -31,4 +18,7 @@ import ( _ "k8s.io/code-generator" _ "sigs.k8s.io/controller-runtime/tools/setup-envtest" _ "sigs.k8s.io/controller-tools/cmd/controller-gen" + _ "sigs.k8s.io/kind" + _ "sigs.k8s.io/kubebuilder/v3/cmd" + _ "sigs.k8s.io/kustomize/kustomize/v5" ) From bd35dbbe6a2303b5a833b5edc2153e0dc2b7bfbe Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 11 Oct 2023 10:19:42 -0500 Subject: [PATCH 26/54] Set v1a2 as the storageVersion This is effectively a noop when the v1a2 FFS is not enabled since our post install configure script already forces the storageVersion back to v1a1 in that case. But when the FFS is enabled this results in the correct storageVersion being set. --- api/v1alpha1/virtualmachine_types.go | 2 +- api/v1alpha1/virtualmachineclass_types.go | 2 +- api/v1alpha1/virtualmachineimage_types.go | 4 ++-- api/v1alpha1/virtualmachinepublishrequest_types.go | 2 +- api/v1alpha1/virtualmachineservice_types.go | 2 +- api/v1alpha1/virtualmachinesetresourcepolicy_types.go | 2 +- api/v1alpha2/virtualmachine_types.go | 2 +- api/v1alpha2/virtualmachineclass_types.go | 2 +- api/v1alpha2/virtualmachineimage_types.go | 4 ++-- api/v1alpha2/virtualmachinepublishrequest_types.go | 2 +- api/v1alpha2/virtualmachineservice_types.go | 2 +- api/v1alpha2/virtualmachinesetresourcepolicy_types.go | 2 +- .../vmoperator.vmware.com_clustervirtualmachineimages.yaml | 4 ++-- .../bases/vmoperator.vmware.com_virtualmachineclasses.yaml | 4 ++-- .../crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml | 4 ++-- .../vmoperator.vmware.com_virtualmachinepublishrequests.yaml | 4 ++-- config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml | 4 ++-- .../bases/vmoperator.vmware.com_virtualmachineservices.yaml | 4 ++-- ...operator.vmware.com_virtualmachinesetresourcepolicies.yaml | 4 ++-- 19 files changed, 28 insertions(+), 28 deletions(-) diff --git a/api/v1alpha1/virtualmachine_types.go b/api/v1alpha1/virtualmachine_types.go index 051efe096..b562f50e7 100644 --- a/api/v1alpha1/virtualmachine_types.go +++ b/api/v1alpha1/virtualmachine_types.go @@ -614,7 +614,7 @@ func (vm *VirtualMachine) SetConditions(conditions Conditions) { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Namespaced,shortName=vm -// +kubebuilder:storageversion +// +kubebuilder:storageversion:false // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Power-State",type="string",JSONPath=".status.powerState" // +kubebuilder:printcolumn:name="Class",type="string",priority=1,JSONPath=".spec.className" diff --git a/api/v1alpha1/virtualmachineclass_types.go b/api/v1alpha1/virtualmachineclass_types.go index 52c9d711d..c8575f205 100644 --- a/api/v1alpha1/virtualmachineclass_types.go +++ b/api/v1alpha1/virtualmachineclass_types.go @@ -133,7 +133,7 @@ type VirtualMachineClassStatus struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=vmclass -// +kubebuilder:storageversion +// +kubebuilder:storageversion:false // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="CPU",type="string",JSONPath=".spec.hardware.cpus" // +kubebuilder:printcolumn:name="Memory",type="string",JSONPath=".spec.hardware.memory" diff --git a/api/v1alpha1/virtualmachineimage_types.go b/api/v1alpha1/virtualmachineimage_types.go index a79c864d5..bc30cd49e 100644 --- a/api/v1alpha1/virtualmachineimage_types.go +++ b/api/v1alpha1/virtualmachineimage_types.go @@ -154,7 +154,7 @@ func (vmImage *VirtualMachineImage) SetConditions(conditions Conditions) { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=vmi;vmimage -// +kubebuilder:storageversion +// +kubebuilder:storageversion:false // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Image-Name",type="string",JSONPath=".status.imageName" // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.productInfo.version" @@ -194,7 +194,7 @@ func (clusterVirtualMachineImage *ClusterVirtualMachineImage) SetConditions(cond // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=cvmi;cvmimage;clustervmi;clustervmimage -// +kubebuilder:storageversion +// +kubebuilder:storageversion:false // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Image-Name",type="string",JSONPath=".status.imageName" // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.productInfo.version" diff --git a/api/v1alpha1/virtualmachinepublishrequest_types.go b/api/v1alpha1/virtualmachinepublishrequest_types.go index 091b7fd39..a1c69e9ec 100644 --- a/api/v1alpha1/virtualmachinepublishrequest_types.go +++ b/api/v1alpha1/virtualmachinepublishrequest_types.go @@ -342,7 +342,7 @@ func (vmpr *VirtualMachinePublishRequest) SetConditions(conditions Conditions) { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Namespaced,shortName=vmpub -// +kubebuilder:storageversion +// +kubebuilder:storageversion:false // +kubebuilder:subresource:status // VirtualMachinePublishRequest defines the information necessary to publish a diff --git a/api/v1alpha1/virtualmachineservice_types.go b/api/v1alpha1/virtualmachineservice_types.go index 381c34bb1..fca195ee8 100644 --- a/api/v1alpha1/virtualmachineservice_types.go +++ b/api/v1alpha1/virtualmachineservice_types.go @@ -132,7 +132,7 @@ type VirtualMachineServiceStatus struct { // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=vmservice -// +kubebuilder:storageversion +// +kubebuilder:storageversion:false // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".spec.type" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" diff --git a/api/v1alpha1/virtualmachinesetresourcepolicy_types.go b/api/v1alpha1/virtualmachinesetresourcepolicy_types.go index ff9288149..6e391f735 100644 --- a/api/v1alpha1/virtualmachinesetresourcepolicy_types.go +++ b/api/v1alpha1/virtualmachinesetresourcepolicy_types.go @@ -55,7 +55,7 @@ type ClusterModuleStatus struct { } // +kubebuilder:object:root=true -// +kubebuilder:storageversion +// +kubebuilder:storageversion:false // +kubebuilder:subresource:status // VirtualMachineSetResourcePolicy is the Schema for the virtualmachinesetresourcepolicies API. diff --git a/api/v1alpha2/virtualmachine_types.go b/api/v1alpha2/virtualmachine_types.go index 90af22976..843253780 100644 --- a/api/v1alpha2/virtualmachine_types.go +++ b/api/v1alpha2/virtualmachine_types.go @@ -459,7 +459,7 @@ type VirtualMachineStatus struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Namespaced,shortName=vm -// +kubebuilder:storageversion:false +// +kubebuilder:storageversion // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Class",type="string",priority=1,JSONPath=".status.class.name" // +kubebuilder:printcolumn:name="Image",type="string",priority=1,JSONPath=".status.image.name" diff --git a/api/v1alpha2/virtualmachineclass_types.go b/api/v1alpha2/virtualmachineclass_types.go index b064df66d..a53500b98 100644 --- a/api/v1alpha2/virtualmachineclass_types.go +++ b/api/v1alpha2/virtualmachineclass_types.go @@ -245,7 +245,7 @@ type VirtualMachineClassStatus struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=vmclass -// +kubebuilder:storageversion:false +// +kubebuilder:storageversion // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="CPU",type="string",JSONPath=".spec.hardware.cpus" // +kubebuilder:printcolumn:name="Memory",type="string",JSONPath=".spec.hardware.memory" diff --git a/api/v1alpha2/virtualmachineimage_types.go b/api/v1alpha2/virtualmachineimage_types.go index fb752062f..c039dc462 100644 --- a/api/v1alpha2/virtualmachineimage_types.go +++ b/api/v1alpha2/virtualmachineimage_types.go @@ -209,7 +209,7 @@ type VirtualMachineImageStatus struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=vmi;vmimage -// +kubebuilder:storageversion:false +// +kubebuilder:storageversion // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Image Name",type="string",JSONPath=".status.name" // +kubebuilder:printcolumn:name="Image Version",type="string",JSONPath=".status.productInfo.version" @@ -246,7 +246,7 @@ type VirtualMachineImageList struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,shortName=cvmi;cvmimage;clustervmi;clustervmimage -// +kubebuilder:storageversion:false +// +kubebuilder:storageversion // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Image Name",type="string",JSONPath=".status.name" // +kubebuilder:printcolumn:name="Image Version",type="string",JSONPath=".status.productInfo.version" diff --git a/api/v1alpha2/virtualmachinepublishrequest_types.go b/api/v1alpha2/virtualmachinepublishrequest_types.go index 3ab47b634..dc7e35730 100644 --- a/api/v1alpha2/virtualmachinepublishrequest_types.go +++ b/api/v1alpha2/virtualmachinepublishrequest_types.go @@ -334,7 +334,7 @@ type VirtualMachinePublishRequestStatus struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Namespaced,shortName=vmpub -// +kubebuilder:storageversion:false +// +kubebuilder:storageversion // +kubebuilder:subresource:status // VirtualMachinePublishRequest defines the information necessary to publish a diff --git a/api/v1alpha2/virtualmachineservice_types.go b/api/v1alpha2/virtualmachineservice_types.go index f8381eea9..674a6010f 100644 --- a/api/v1alpha2/virtualmachineservice_types.go +++ b/api/v1alpha2/virtualmachineservice_types.go @@ -139,7 +139,7 @@ type VirtualMachineServiceStatus struct { // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=vmservice -// +kubebuilder:storageversion:false +// +kubebuilder:storageversion // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".spec.type" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" diff --git a/api/v1alpha2/virtualmachinesetresourcepolicy_types.go b/api/v1alpha2/virtualmachinesetresourcepolicy_types.go index 582eb2e40..0d7e9e62d 100644 --- a/api/v1alpha2/virtualmachinesetresourcepolicy_types.go +++ b/api/v1alpha2/virtualmachinesetresourcepolicy_types.go @@ -47,7 +47,7 @@ type VSphereClusterModuleStatus struct { } // +kubebuilder:object:root=true -// +kubebuilder:storageversion:false +// +kubebuilder:storageversion // +kubebuilder:subresource:status // VirtualMachineSetResourcePolicy is the Schema for the virtualmachinesetresourcepolicies API. diff --git a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml index 50043573c..315800fd0 100644 --- a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml @@ -270,7 +270,7 @@ spec: type: object type: object served: true - storage: true + storage: false subresources: status: {} - additionalPrinterColumns: @@ -513,6 +513,6 @@ spec: type: object type: object served: true - storage: false + storage: true subresources: status: {} diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineclasses.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineclasses.yaml index c536b2f13..89897f23d 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineclasses.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineclasses.yaml @@ -215,7 +215,7 @@ spec: type: object type: object served: true - storage: true + storage: false subresources: status: {} - additionalPrinterColumns: @@ -499,6 +499,6 @@ spec: type: object type: object served: true - storage: false + storage: true subresources: status: {} diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml index 3e314548b..fb7c0ee1e 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml @@ -270,7 +270,7 @@ spec: type: object type: object served: true - storage: true + storage: false subresources: status: {} - additionalPrinterColumns: @@ -516,6 +516,6 @@ spec: type: object type: object served: true - storage: false + storage: true subresources: status: {} diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml index 7fb9687b2..a89b36c94 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml @@ -292,7 +292,7 @@ spec: type: object type: object served: true - storage: true + storage: false subresources: status: {} - name: v1alpha2 @@ -595,6 +595,6 @@ spec: type: object type: object served: true - storage: false + storage: true subresources: status: {} diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml index 06c3acfbd..ee0acb753 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml @@ -593,7 +593,7 @@ spec: type: object type: object served: true - storage: true + storage: false subresources: status: {} - additionalPrinterColumns: @@ -2709,6 +2709,6 @@ spec: type: object type: object served: true - storage: false + storage: true subresources: status: {} diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml index b5fe201ba..fd956acad 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml @@ -175,7 +175,7 @@ spec: type: object type: object served: true - storage: true + storage: false subresources: status: {} - additionalPrinterColumns: @@ -323,6 +323,6 @@ spec: type: object type: object served: true - storage: false + storage: true subresources: status: {} diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachinesetresourcepolicies.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachinesetresourcepolicies.yaml index 3cf046ae9..ce2e25a8c 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachinesetresourcepolicies.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachinesetresourcepolicies.yaml @@ -124,7 +124,7 @@ spec: type: object type: object served: true - storage: true + storage: false subresources: status: {} - name: v1alpha2 @@ -222,6 +222,6 @@ spec: type: object type: object served: true - storage: false + storage: true subresources: status: {} From 529883c926097d9cfc1fae9ceb7665b49900a262 Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Fri, 13 Oct 2023 14:24:44 -0700 Subject: [PATCH 27/54] Include the v1a2 webconsole CRD in the combined CRD manifest (#247) --- config/crd/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 707333155..8eac32599 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -14,7 +14,7 @@ resources: - bases/vmoperator.vmware.com_virtualmachineimages.yaml - bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml - bases/vmoperator.vmware.com_webconsolerequests.yaml -#- bases/vmoperator.vmware.com_virtualmachinewebconsolerequests.yaml +- bases/vmoperator.vmware.com_virtualmachinewebconsolerequests.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: From 76e606ac80d97e6ce6178f2c883a0aff5c233bc2 Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Mon, 16 Oct 2023 15:37:38 -0400 Subject: [PATCH 28/54] Add friendly image name support in v1a2 and improve duplicate check (#243) This patch updates the VM mutation webhook to take both namespace and cluster scope images into consideration when resolving the VM image resource from the specified friendly name. It also adds the corresponding implementation in v1a2. --- docs/concepts/images/vm-image.md | 2 +- .../mutation/virtualmachine_mutator.go | 31 +++- .../virtualmachine_mutator_intg_test.go | 2 +- .../virtualmachine_mutator_unit_test.go | 54 ++++--- .../mutation/virtualmachine_mutator.go | 94 +++++++++++- .../virtualmachine_mutator_intg_test.go | 30 ++++ .../virtualmachine_mutator_unit_test.go | 139 ++++++++++++++++++ 7 files changed, 324 insertions(+), 28 deletions(-) diff --git a/docs/concepts/images/vm-image.md b/docs/concepts/images/vm-image.md index c080ecf18..a80867177 100644 --- a/docs/concepts/images/vm-image.md +++ b/docs/concepts/images/vm-image.md @@ -44,7 +44,7 @@ For example, if `vmi-0a0044d7c690bcbea` refers to an image with a friendly name * There is no other `VirtualMachineImage` in the same namespace with that friendly name. * There is no other `ClusterVirtualMachineImage` with the same friendly name. -If the friendly name unambiguously resolves to the distinct, VM image `vmi-0a0044d7c690bcbea`, then a mutation webhook replaces `spec.imageName: photonos-5-x64` with `spec.imageName: vmi-0a0044d7c690bcbea`. +If the friendly name unambiguously resolves to the distinct, VM image `vmi-0a0044d7c690bcbea`, then a mutation webhook replaces `spec.imageName: photonos-5-x64` with `spec.imageName: vmi-0a0044d7c690bcbea`. If the friendly name resolves to multiple or no VM images, then the mutation webhook denies the request and outputs an error message accordingly. ## Recommended Images diff --git a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go index 82e23f950..c66babe65 100644 --- a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go +++ b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator.go @@ -199,6 +199,7 @@ func ResolveImageName( return false, nil } + var determinedImageName string // Check if a single namespace scope image exists by the status name. vmiList := &vmopv1.VirtualMachineImageList{} if err := c.List(ctx, vmiList, client.InNamespace(vm.Namespace), @@ -208,9 +209,13 @@ func ResolveImageName( ); err != nil { return false, err } - if len(vmiList.Items) == 1 { - vm.Spec.ImageName = vmiList.Items[0].Name - return true, nil + switch len(vmiList.Items) { + case 0: + break + case 1: + determinedImageName = vmiList.Items[0].Name + default: + return false, errors.Errorf("multiple VM images exist for %q in namespace scope", imageName) } // Check if a single cluster scope image exists by the status name. @@ -220,12 +225,24 @@ func ResolveImageName( }); err != nil { return false, err } - if len(cvmiList.Items) == 1 { - vm.Spec.ImageName = cvmiList.Items[0].Name - return true, nil + switch len(cvmiList.Items) { + case 0: + break + case 1: + if determinedImageName != "" { + return false, errors.Errorf("multiple VM images exist for %q in namespace and cluster scope", imageName) + } + determinedImageName = cvmiList.Items[0].Name + default: + return false, errors.Errorf("multiple VM images exist for %q in cluster scope", imageName) + } + + if determinedImageName == "" { + return false, errors.Errorf("no VM image exists for %q in namespace or cluster scope", imageName) } - return false, errors.Errorf("no single VM image exists for %q", imageName) + vm.Spec.ImageName = determinedImageName + return true, nil } // SetNextRestartTime sets spec.nextRestartTime for a VM if the field's diff --git a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_intg_test.go b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_intg_test.go index 215865f06..b0be57b18 100644 --- a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_intg_test.go +++ b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_intg_test.go @@ -182,7 +182,7 @@ func intgTestsMutating() { When("Creating VirtualMachine", func() { - When("When VM ImageName is already a vmi resource name", func() { + When("VM ImageName is already a vmi resource name", func() { BeforeEach(func() { vm.Spec.ImageName = "vmi-123" diff --git a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_unit_test.go b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_unit_test.go index 2a0b937ba..2d530ae26 100644 --- a/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha1/mutation/virtualmachine_mutator_unit_test.go @@ -248,6 +248,10 @@ func unitTestsMutating() { }) Describe(("ResolveImageName"), func() { + const ( + dupImageStatusName = "dup-status-name" + uniqueImageStatusName = "unique-status-name" + ) BeforeEach(func() { // Replace the client with a fake client that has the index of VM images. @@ -279,62 +283,77 @@ func unitTestsMutating() { }) It("Should not mutate ImageName", func() { - oldVM := ctx.vm.DeepCopy() mutated, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) Expect(err).ToNot(HaveOccurred()) Expect(mutated).To(BeFalse()) - Expect(ctx.vm.Spec.ImageName).Should(Equal(oldVM.Spec.ImageName)) + Expect(ctx.vm.Spec.ImageName).Should(Equal("vmi-xxx")) }) }) Context("When VM ImageName is set to a status name matching multiple namespace scope images", func() { BeforeEach(func() { - dupStatusName := "dup-status-name" vmi1 := builder.DummyVirtualMachineImage("vmi-1") - vmi1.Status.ImageName = dupStatusName + vmi1.Status.ImageName = dupImageStatusName vmi2 := builder.DummyVirtualMachineImage("vmi-2") - vmi2.Status.ImageName = dupStatusName + vmi2.Status.ImageName = dupImageStatusName Expect(ctx.Client.Create(ctx, vmi1)).To(Succeed()) Expect(ctx.Client.Create(ctx, vmi2)).To(Succeed()) - ctx.vm.Spec.ImageName = dupStatusName + ctx.vm.Spec.ImageName = dupImageStatusName }) It("Should return an error", func() { _, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal("no single VM image exists for \"dup-status-name\"")) + Expect(err.Error()).To(Equal("multiple VM images exist for \"dup-status-name\" in namespace scope")) }) }) Context("When VM ImageName is set to a status name matching multiple cluster scope images", func() { BeforeEach(func() { - dupStatusName := "dup-status-name" cvmi1 := builder.DummyClusterVirtualMachineImage("cvmi-1") - cvmi1.Status.ImageName = dupStatusName + cvmi1.Status.ImageName = dupImageStatusName cvmi2 := builder.DummyClusterVirtualMachineImage("cvmi-2") - cvmi2.Status.ImageName = dupStatusName + cvmi2.Status.ImageName = dupImageStatusName Expect(ctx.Client.Create(ctx, cvmi1)).To(Succeed()) Expect(ctx.Client.Create(ctx, cvmi2)).To(Succeed()) - ctx.vm.Spec.ImageName = dupStatusName + ctx.vm.Spec.ImageName = dupImageStatusName + }) + + It("Should return an error", func() { + _, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("multiple VM images exist for \"dup-status-name\" in cluster scope")) + }) + }) + + Context("When VM ImageName is set to a status name matching one namespace and one cluster scope images", func() { + + BeforeEach(func() { + vmi := builder.DummyVirtualMachineImage("vmi-123") + vmi.Status.ImageName = dupImageStatusName + cvmi := builder.DummyClusterVirtualMachineImage("cvmi-123") + cvmi.Status.ImageName = dupImageStatusName + Expect(ctx.Client.Create(ctx, vmi)).To(Succeed()) + Expect(ctx.Client.Create(ctx, cvmi)).To(Succeed()) + ctx.vm.Spec.ImageName = dupImageStatusName }) It("Should return an error", func() { _, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal("no single VM image exists for \"dup-status-name\"")) + Expect(err.Error()).To(Equal("multiple VM images exist for \"dup-status-name\" in namespace and cluster scope")) }) }) Context("When VM ImageName is set to a status name matching a single namespace scope image", func() { BeforeEach(func() { - uniqueStatusName := "unique-status-name" vmi := builder.DummyVirtualMachineImage("vmi-123") - vmi.Status.ImageName = uniqueStatusName + vmi.Status.ImageName = uniqueImageStatusName Expect(ctx.Client.Create(ctx, vmi)).To(Succeed()) - ctx.vm.Spec.ImageName = uniqueStatusName + ctx.vm.Spec.ImageName = uniqueImageStatusName }) It("Should mutate ImageName to the resource name of the namespace scope image", func() { @@ -348,11 +367,10 @@ func unitTestsMutating() { Context("When VM ImageName is set to a status name matching a single cluster scope image", func() { BeforeEach(func() { - uniqueStatusName := "unique-status-name" cvmi := builder.DummyClusterVirtualMachineImage("vmi-123") - cvmi.Status.ImageName = uniqueStatusName + cvmi.Status.ImageName = uniqueImageStatusName Expect(ctx.Client.Create(ctx, cvmi)).To(Succeed()) - ctx.vm.Spec.ImageName = uniqueStatusName + ctx.vm.Spec.ImageName = uniqueImageStatusName }) It("Should mutate ImageName to the resource name of the cluster scope image", func() { diff --git a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go index 5ab6c2cdd..612e1a9c0 100644 --- a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go +++ b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go @@ -22,6 +22,7 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/pkg/builder" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" ) const ( @@ -31,9 +32,36 @@ const ( // +kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha2-virtualmachine,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,verbs=create;update,versions=v1alpha2,name=default.mutating.virtualmachine.v1alpha2.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachine,verbs=get;list // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachine/status,verbs=get +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineimages,verbs=get;list;watch +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineimages/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=clustervirtualmachineimages,verbs=get;list;watch +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=clustervirtualmachineimages/status,verbs=get;list;watch // AddToManager adds the webhook to the provided manager. func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + // Index the VirtualMachineImage and ClusterVirtualMachineImage objects by + // status.name field to allow efficient querying in ResolveImageName(). + if err := mgr.GetFieldIndexer().IndexField( + ctx, + &vmopv1.VirtualMachineImage{}, + "status.name", + func(rawObj client.Object) []string { + vmi := rawObj.(*vmopv1.VirtualMachineImage) + return []string{vmi.Status.Name} + }); err != nil { + return err + } + if err := mgr.GetFieldIndexer().IndexField( + ctx, + &vmopv1.ClusterVirtualMachineImage{}, + "status.name", + func(rawObj client.Object) []string { + cvmi := rawObj.(*vmopv1.ClusterVirtualMachineImage) + return []string{cvmi.Status.Name} + }); err != nil { + return err + } + hook, err := builder.NewMutatingWebhook(ctx, mgr, webHookName, NewMutator(mgr.GetClient())) if err != nil { return errors.Wrapf(err, "failed to create mutation webhook") @@ -70,8 +98,13 @@ func (m mutator) Mutate(ctx *context.WebhookRequestContext) admission.Response { original := vm modified := original.DeepCopy() - //nolint:gocritic switch ctx.Op { + case admissionv1.Create: + if mutated, err := ResolveImageName(ctx, m.client, modified); err != nil { + return admission.Denied(err.Error()) + } else if mutated { + wasMutated = true + } case admissionv1.Update: oldVM, err := m.vmFromUnstructured(ctx.OldObj) if err != nil { @@ -142,3 +175,62 @@ func SetNextRestartTime( newVM.Spec.NextRestartTime, `may only be set to "now"`) } + +// ResolveImageName mutates the vm.spec.imageName if it's not set to a vmi name +// and there is a single namespace or cluster scope image with that status name. +func ResolveImageName( + ctx *context.WebhookRequestContext, + c client.Client, + vm *vmopv1.VirtualMachine) (bool, error) { + // Return early if the VM image name is empty or already set to a vmi name. + imageName := vm.Spec.ImageName + if imageName == "" || !lib.IsWCPVMImageRegistryEnabled() || + strings.HasPrefix(imageName, "vmi-") { + return false, nil + } + + var determinedImageName string + // Check if a single namespace scope image exists by the status name. + vmiList := &vmopv1.VirtualMachineImageList{} + if err := c.List(ctx, vmiList, client.InNamespace(vm.Namespace), + client.MatchingFields{ + "status.name": imageName, + }, + ); err != nil { + return false, err + } + switch len(vmiList.Items) { + case 0: + break + case 1: + determinedImageName = vmiList.Items[0].Name + default: + return false, errors.Errorf("multiple VM images exist for %q in namespace scope", imageName) + } + + // Check if a single cluster scope image exists by the status name. + cvmiList := &vmopv1.ClusterVirtualMachineImageList{} + if err := c.List(ctx, cvmiList, client.MatchingFields{ + "status.name": imageName, + }); err != nil { + return false, err + } + switch len(cvmiList.Items) { + case 0: + break + case 1: + if determinedImageName != "" { + return false, errors.Errorf("multiple VM images exist for %q in namespace and cluster scope", imageName) + } + determinedImageName = cvmiList.Items[0].Name + default: + return false, errors.Errorf("multiple VM images exist for %q in cluster scope", imageName) + } + + if determinedImageName == "" { + return false, errors.Errorf("no VM image exists for %q in namespace or cluster scope", imageName) + } + + vm.Spec.ImageName = determinedImageName + return true, nil +} diff --git a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_intg_test.go b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_intg_test.go index c3b02546b..99ecb2c3d 100644 --- a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_intg_test.go +++ b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_intg_test.go @@ -4,6 +4,7 @@ package mutation_test import ( + "os" "time" . "github.com/onsi/ginkgo" @@ -14,6 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/test/builder" ) @@ -148,4 +150,32 @@ func intgTestsMutating() { }) }) }) + + Context("ResolveImageName", func() { + + BeforeEach(func() { + Expect(os.Setenv(lib.VMImageRegistryFSS, lib.TrueString)).To(Succeed()) + }) + + AfterEach(func() { + Expect(os.Unsetenv(lib.VMImageRegistryFSS)).To(Succeed()) + }) + + When("Creating VirtualMachine", func() { + + When("VM ImageName is already a vmi resource name", func() { + + BeforeEach(func() { + ctx.vm.Spec.ImageName = "vmi-123" + }) + + It("Should not mutate ImageName", func() { + Expect(ctx.Client.Create(ctx, ctx.vm)).To(Succeed()) + modified := &vmopv1.VirtualMachine{} + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(ctx.vm), modified)).Should(Succeed()) + Expect(modified.Spec.ImageName).Should(Equal("vmi-123")) + }) + }) + }) + }) } diff --git a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_unit_test.go b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_unit_test.go index 77eecc8cd..b3515fbeb 100644 --- a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_unit_test.go @@ -4,6 +4,7 @@ package mutation_test import ( + "os" "time" . "github.com/onsi/ginkgo" @@ -12,8 +13,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/test/builder" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha2/mutation" ) @@ -222,4 +226,139 @@ func unitTestsMutating() { ) }) }) + + Describe(("ResolveImageName"), func() { + const ( + dupImageStatusName = "dup-status-name" + uniqueImageStatusName = "unique-status-name" + ) + + BeforeEach(func() { + // Replace the client with a fake client that has the index of VM images. + ctx.Client = fake.NewClientBuilder().WithScheme(builder.NewScheme()). + WithIndex( + &vmopv1.VirtualMachineImage{}, + "status.name", + func(rawObj client.Object) []string { + image := rawObj.(*vmopv1.VirtualMachineImage) + return []string{image.Status.Name} + }). + WithIndex(&vmopv1.ClusterVirtualMachineImage{}, + "status.name", + func(rawObj client.Object) []string { + image := rawObj.(*vmopv1.ClusterVirtualMachineImage) + return []string{image.Status.Name} + }).Build() + Expect(os.Setenv(lib.VMImageRegistryFSS, lib.TrueString)).To(Succeed()) + }) + + AfterEach(func() { + Expect(os.Unsetenv(lib.VMImageRegistryFSS)).To(Succeed()) + }) + + Context("When VM ImageName is set to vmi resource name", func() { + + BeforeEach(func() { + ctx.vm.Spec.ImageName = "vmi-xxx" + }) + + It("Should not mutate ImageName", func() { + mutated, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) + Expect(err).ToNot(HaveOccurred()) + Expect(mutated).To(BeFalse()) + Expect(ctx.vm.Spec.ImageName).Should(Equal("vmi-xxx")) + }) + }) + + Context("When VM ImageName is set to a status name matching multiple namespace scope images", func() { + + BeforeEach(func() { + vmi1 := builder.DummyVirtualMachineImageA2("vmi-1") + vmi1.Status.Name = dupImageStatusName + vmi2 := builder.DummyVirtualMachineImageA2("vmi-2") + vmi2.Status.Name = dupImageStatusName + Expect(ctx.Client.Create(ctx, vmi1)).To(Succeed()) + Expect(ctx.Client.Create(ctx, vmi2)).To(Succeed()) + ctx.vm.Spec.ImageName = dupImageStatusName + }) + + It("Should return an error", func() { + _, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("multiple VM images exist for \"dup-status-name\" in namespace scope")) + }) + }) + + Context("When VM ImageName is set to a status name matching multiple cluster scope images", func() { + + BeforeEach(func() { + cvmi1 := builder.DummyClusterVirtualMachineImageA2("cvmi-1") + cvmi1.Status.Name = dupImageStatusName + cvmi2 := builder.DummyClusterVirtualMachineImageA2("cvmi-2") + cvmi2.Status.Name = dupImageStatusName + Expect(ctx.Client.Create(ctx, cvmi1)).To(Succeed()) + Expect(ctx.Client.Create(ctx, cvmi2)).To(Succeed()) + ctx.vm.Spec.ImageName = dupImageStatusName + }) + + It("Should return an error", func() { + _, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("multiple VM images exist for \"dup-status-name\" in cluster scope")) + }) + }) + + Context("When VM ImageName is set to a status name matching one namespace and one cluster scope images", func() { + + BeforeEach(func() { + vmi := builder.DummyVirtualMachineImageA2("vmi-123") + vmi.Status.Name = dupImageStatusName + cvmi := builder.DummyClusterVirtualMachineImageA2("cvmi-123") + cvmi.Status.Name = dupImageStatusName + Expect(ctx.Client.Create(ctx, vmi)).To(Succeed()) + Expect(ctx.Client.Create(ctx, cvmi)).To(Succeed()) + ctx.vm.Spec.ImageName = dupImageStatusName + }) + + It("Should return an error", func() { + _, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("multiple VM images exist for \"dup-status-name\" in namespace and cluster scope")) + }) + }) + + Context("When VM ImageName is set to a status name matching a single namespace scope image", func() { + + BeforeEach(func() { + vmi := builder.DummyVirtualMachineImageA2("vmi-123") + vmi.Status.Name = uniqueImageStatusName + Expect(ctx.Client.Create(ctx, vmi)).To(Succeed()) + ctx.vm.Spec.ImageName = uniqueImageStatusName + }) + + It("Should mutate ImageName to the resource name of the namespace scope image", func() { + mutated, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) + Expect(err).ToNot(HaveOccurred()) + Expect(mutated).To(BeTrue()) + Expect(ctx.vm.Spec.ImageName).Should(Equal("vmi-123")) + }) + }) + + Context("When VM ImageName is set to a status name matching a single cluster scope image", func() { + + BeforeEach(func() { + cvmi := builder.DummyClusterVirtualMachineImageA2("vmi-123") + cvmi.Status.Name = uniqueImageStatusName + Expect(ctx.Client.Create(ctx, cvmi)).To(Succeed()) + ctx.vm.Spec.ImageName = uniqueImageStatusName + }) + + It("Should mutate ImageName to the resource name of the cluster scope image", func() { + mutated, err := mutation.ResolveImageName(&ctx.WebhookRequestContext, ctx.Client, ctx.vm) + Expect(err).ToNot(HaveOccurred()) + Expect(mutated).To(BeTrue()) + Expect(ctx.vm.Spec.ImageName).Should(Equal("vmi-123")) + }) + }) + }) } From 0b17895e4c7adac2682df42ea12ff65d7df613c7 Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Mon, 16 Oct 2023 15:40:01 -0400 Subject: [PATCH 29/54] Rename IMAGE-NAME to HUMAN-READABLE-NAME in VM Image printer column (#215) This patch updates the printer column of VirtualMachine Image (in both v1a1 and v1a2) to rename "Image Name" to "Human Readable Name". The previous "Image Name" can be confusing with the `spec.imageName` in the VirtualMachine type. --- api/v1alpha1/virtualmachineimage_types.go | 4 ++-- api/v1alpha2/virtualmachineimage_types.go | 4 ++-- .../vmoperator.vmware.com_clustervirtualmachineimages.yaml | 4 ++-- .../crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/virtualmachineimage_types.go b/api/v1alpha1/virtualmachineimage_types.go index bc30cd49e..c482734e6 100644 --- a/api/v1alpha1/virtualmachineimage_types.go +++ b/api/v1alpha1/virtualmachineimage_types.go @@ -156,7 +156,7 @@ func (vmImage *VirtualMachineImage) SetConditions(conditions Conditions) { // +kubebuilder:resource:scope=Cluster,shortName=vmi;vmimage // +kubebuilder:storageversion:false // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Image-Name",type="string",JSONPath=".status.imageName" +// +kubebuilder:printcolumn:name="Human-Readable-Name",type="string",JSONPath=".status.imageName" // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.productInfo.version" // +kubebuilder:printcolumn:name="Os-Type",type="string",JSONPath=".spec.osInfo.type" // +kubebuilder:printcolumn:name="Format",type="string",JSONPath=".spec.type" @@ -196,7 +196,7 @@ func (clusterVirtualMachineImage *ClusterVirtualMachineImage) SetConditions(cond // +kubebuilder:resource:scope=Cluster,shortName=cvmi;cvmimage;clustervmi;clustervmimage // +kubebuilder:storageversion:false // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Image-Name",type="string",JSONPath=".status.imageName" +// +kubebuilder:printcolumn:name="Human-Readable-Name",type="string",JSONPath=".status.imageName" // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.productInfo.version" // +kubebuilder:printcolumn:name="Os-Type",type="string",JSONPath=".spec.osInfo.type" // +kubebuilder:printcolumn:name="Format",type="string",JSONPath=".spec.type" diff --git a/api/v1alpha2/virtualmachineimage_types.go b/api/v1alpha2/virtualmachineimage_types.go index c039dc462..673807de7 100644 --- a/api/v1alpha2/virtualmachineimage_types.go +++ b/api/v1alpha2/virtualmachineimage_types.go @@ -211,7 +211,7 @@ type VirtualMachineImageStatus struct { // +kubebuilder:resource:scope=Cluster,shortName=vmi;vmimage // +kubebuilder:storageversion // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Image Name",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="Human Readable Name",type="string",JSONPath=".status.name" // +kubebuilder:printcolumn:name="Image Version",type="string",JSONPath=".status.productInfo.version" // +kubebuilder:printcolumn:name="OS Name",type="string",JSONPath=".status.osInfo.type" // +kubebuilder:printcolumn:name="OS Version",type="string",JSONPath=".status.osInfo.version" @@ -248,7 +248,7 @@ type VirtualMachineImageList struct { // +kubebuilder:resource:scope=Cluster,shortName=cvmi;cvmimage;clustervmi;clustervmimage // +kubebuilder:storageversion // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Image Name",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="Human Readable Name",type="string",JSONPath=".status.name" // +kubebuilder:printcolumn:name="Image Version",type="string",JSONPath=".status.productInfo.version" // +kubebuilder:printcolumn:name="OS Name",type="string",JSONPath=".status.osInfo.type" // +kubebuilder:printcolumn:name="OS Version",type="string",JSONPath=".status.osInfo.version" diff --git a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml index 315800fd0..d92ab3693 100644 --- a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml @@ -22,7 +22,7 @@ spec: versions: - additionalPrinterColumns: - jsonPath: .status.imageName - name: Image-Name + name: Human-Readable-Name type: string - jsonPath: .spec.productInfo.version name: Version @@ -275,7 +275,7 @@ spec: status: {} - additionalPrinterColumns: - jsonPath: .status.name - name: Image Name + name: Human Readable Name type: string - jsonPath: .status.productInfo.version name: Image Version diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml index fb7c0ee1e..d3acccb2f 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml @@ -20,7 +20,7 @@ spec: versions: - additionalPrinterColumns: - jsonPath: .status.imageName - name: Image-Name + name: Human-Readable-Name type: string - jsonPath: .spec.productInfo.version name: Version @@ -275,7 +275,7 @@ spec: status: {} - additionalPrinterColumns: - jsonPath: .status.name - name: Image Name + name: Human Readable Name type: string - jsonPath: .status.productInfo.version name: Image Version From 19d7d2b7b7f5b94cb13db846b5ec11a225783dcc Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Mon, 16 Oct 2023 17:55:58 -0400 Subject: [PATCH 30/54] Adopt image display name in API and docs --- api/v1alpha1/virtualmachineimage_types.go | 6 +- .../virtualmachinepublishrequest_types.go | 2 +- api/v1alpha2/virtualmachineimage_types.go | 6 +- ...mware.com_clustervirtualmachineimages.yaml | 9 +- ...rator.vmware.com_virtualmachineimages.yaml | 9 +- ...are.com_virtualmachinepublishrequests.yaml | 8 +- docs/concepts/images/vm-image.md | 14 +-- docs/ref/api/v1alpha1.md | 25 ++++- docs/ref/api/v1alpha2.md | 93 ++++++++++++++----- 9 files changed, 119 insertions(+), 53 deletions(-) diff --git a/api/v1alpha1/virtualmachineimage_types.go b/api/v1alpha1/virtualmachineimage_types.go index c482734e6..a676defca 100644 --- a/api/v1alpha1/virtualmachineimage_types.go +++ b/api/v1alpha1/virtualmachineimage_types.go @@ -114,7 +114,7 @@ type VirtualMachineImageStatus struct { // Deprecated PowerState string `json:"powerState,omitempty"` - // ImageName describes the display name of this VirtualMachineImage. + // ImageName describes the display name of this image. // +optional ImageName string `json:"imageName,omitempty"` @@ -156,7 +156,7 @@ func (vmImage *VirtualMachineImage) SetConditions(conditions Conditions) { // +kubebuilder:resource:scope=Cluster,shortName=vmi;vmimage // +kubebuilder:storageversion:false // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Human-Readable-Name",type="string",JSONPath=".status.imageName" +// +kubebuilder:printcolumn:name="Display-Name",type="string",JSONPath=".status.imageName" // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.productInfo.version" // +kubebuilder:printcolumn:name="Os-Type",type="string",JSONPath=".spec.osInfo.type" // +kubebuilder:printcolumn:name="Format",type="string",JSONPath=".spec.type" @@ -196,7 +196,7 @@ func (clusterVirtualMachineImage *ClusterVirtualMachineImage) SetConditions(cond // +kubebuilder:resource:scope=Cluster,shortName=cvmi;cvmimage;clustervmi;clustervmimage // +kubebuilder:storageversion:false // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Human-Readable-Name",type="string",JSONPath=".status.imageName" +// +kubebuilder:printcolumn:name="Display-Name",type="string",JSONPath=".status.imageName" // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".spec.productInfo.version" // +kubebuilder:printcolumn:name="Os-Type",type="string",JSONPath=".spec.osInfo.type" // +kubebuilder:printcolumn:name="Format",type="string",JSONPath=".spec.type" diff --git a/api/v1alpha1/virtualmachinepublishrequest_types.go b/api/v1alpha1/virtualmachinepublishrequest_types.go index a1c69e9ec..d75dc818e 100644 --- a/api/v1alpha1/virtualmachinepublishrequest_types.go +++ b/api/v1alpha1/virtualmachinepublishrequest_types.go @@ -134,7 +134,7 @@ type VirtualMachinePublishRequestSource struct { // VirtualMachinePublishRequestTargetItem is the item part of a // publication request's target. type VirtualMachinePublishRequestTargetItem struct { - // Name is the name of the published object. + // Name is the display name of the published object. // // If the spec.target.location.apiVersion equals // imageregistry.vmware.com/v1alpha1 and the spec.target.location.kind diff --git a/api/v1alpha2/virtualmachineimage_types.go b/api/v1alpha2/virtualmachineimage_types.go index 673807de7..c92d416f6 100644 --- a/api/v1alpha2/virtualmachineimage_types.go +++ b/api/v1alpha2/virtualmachineimage_types.go @@ -132,7 +132,7 @@ type VirtualMachineImageSpec struct { // VirtualMachineImageStatus defines the observed state of VirtualMachineImage. type VirtualMachineImageStatus struct { - // Name describes the observed, "friendly" name for this image. + // Name describes the display name of this image. // // +optional Name string `json:"name,omitempty"` @@ -211,7 +211,7 @@ type VirtualMachineImageStatus struct { // +kubebuilder:resource:scope=Cluster,shortName=vmi;vmimage // +kubebuilder:storageversion // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Human Readable Name",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="Display Name",type="string",JSONPath=".status.name" // +kubebuilder:printcolumn:name="Image Version",type="string",JSONPath=".status.productInfo.version" // +kubebuilder:printcolumn:name="OS Name",type="string",JSONPath=".status.osInfo.type" // +kubebuilder:printcolumn:name="OS Version",type="string",JSONPath=".status.osInfo.version" @@ -248,7 +248,7 @@ type VirtualMachineImageList struct { // +kubebuilder:resource:scope=Cluster,shortName=cvmi;cvmimage;clustervmi;clustervmimage // +kubebuilder:storageversion // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Human Readable Name",type="string",JSONPath=".status.name" +// +kubebuilder:printcolumn:name="Display Name",type="string",JSONPath=".status.name" // +kubebuilder:printcolumn:name="Image Version",type="string",JSONPath=".status.productInfo.version" // +kubebuilder:printcolumn:name="OS Name",type="string",JSONPath=".status.osInfo.type" // +kubebuilder:printcolumn:name="OS Version",type="string",JSONPath=".status.osInfo.version" diff --git a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml index d92ab3693..f018da591 100644 --- a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml @@ -22,7 +22,7 @@ spec: versions: - additionalPrinterColumns: - jsonPath: .status.imageName - name: Human-Readable-Name + name: Display-Name type: string - jsonPath: .spec.productInfo.version name: Version @@ -251,7 +251,7 @@ spec: eg: bios, efi.' type: string imageName: - description: ImageName describes the display name of this VirtualMachineImage. + description: ImageName describes the display name of this image. type: string imageSupported: description: 'ImageSupported indicates whether the VirtualMachineImage @@ -275,7 +275,7 @@ spec: status: {} - additionalPrinterColumns: - jsonPath: .status.name - name: Human Readable Name + name: Display Name type: string - jsonPath: .status.productInfo.version name: Image Version @@ -434,8 +434,7 @@ spec: format: int32 type: integer name: - description: Name describes the observed, "friendly" name for this - image. + description: Name describes the display name of this image. type: string osInfo: description: "OSInfo describes the observed operating system information diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml index d3acccb2f..94e957065 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml @@ -20,7 +20,7 @@ spec: versions: - additionalPrinterColumns: - jsonPath: .status.imageName - name: Human-Readable-Name + name: Display-Name type: string - jsonPath: .spec.productInfo.version name: Version @@ -251,7 +251,7 @@ spec: eg: bios, efi.' type: string imageName: - description: ImageName describes the display name of this VirtualMachineImage. + description: ImageName describes the display name of this image. type: string imageSupported: description: 'ImageSupported indicates whether the VirtualMachineImage @@ -275,7 +275,7 @@ spec: status: {} - additionalPrinterColumns: - jsonPath: .status.name - name: Human Readable Name + name: Display Name type: string - jsonPath: .status.productInfo.version name: Image Version @@ -437,8 +437,7 @@ spec: format: int32 type: integer name: - description: Name describes the observed, "friendly" name for this - image. + description: Name describes the display name of this image. type: string osInfo: description: "OSInfo describes the observed operating system information diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml index a89b36c94..f527493e1 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml @@ -90,8 +90,8 @@ spec: published object. type: string name: - description: "Name is the name of the published object. \n - If the spec.target.location.apiVersion equals imageregistry.vmware.com/v1alpha1 + description: "Name is the display name of the published object. + \n If the spec.target.location.apiVersion equals imageregistry.vmware.com/v1alpha1 and the spec.target.location.kind equals ContentLibrary, then this should be the name that will show up in vCenter Content Library, not the custom resource name in the namespace. @@ -257,8 +257,8 @@ spec: published object. type: string name: - description: "Name is the name of the published object. \n - If the spec.target.location.apiVersion equals imageregistry.vmware.com/v1alpha1 + description: "Name is the display name of the published object. + \n If the spec.target.location.apiVersion equals imageregistry.vmware.com/v1alpha1 and the spec.target.location.kind equals ContentLibrary, then this should be the name that will show up in vCenter Content Library, not the custom resource name in the namespace. diff --git a/docs/concepts/images/vm-image.md b/docs/concepts/images/vm-image.md index a80867177..0872c7278 100644 --- a/docs/concepts/images/vm-image.md +++ b/docs/concepts/images/vm-image.md @@ -33,18 +33,18 @@ For example, if the Content Library item's UUID is `e1968c25-dd84-4506-8dc7-9bea ## Name Resolution -When a `VirtualMachine` resource's field `spec.imageName` is set to a VMI ID, the value is resolved to the `VirtualMachineImage` or `ClusterVirtualMachineImage` with that name. It is also possible to specify images based on their _friendly_ name. +When a `VirtualMachine` resource's field `spec.imageName` is set to a VMI ID, the value is resolved to the `VirtualMachineImage` or `ClusterVirtualMachineImage` with that name. It is also possible to specify images based on their _display_ name. -!!! warning "Friendly name resolution" +!!! warning "Display name resolution" - Please note that while resolving VM images based on their friendly name was merged into VM Operator with [github.com/vmware-tanzu/vm-operator#214](https://github.com/vmware-tanzu/vm-operator/issues/214), the feature is not yet part of a shipping vSphere release. + Please note that while resolving VM images based on their display name was merged into VM Operator with [github.com/vmware-tanzu/vm-operator#214](https://github.com/vmware-tanzu/vm-operator/issues/214), the feature is not yet part of a shipping vSphere release. -For example, if `vmi-0a0044d7c690bcbea` refers to an image with a friendly name of `photonos-5-x64`, then a user could also specify that value for `spec.imageName` as long as the following is true: +For example, if `vmi-0a0044d7c690bcbea` refers to an image with a display name of `photonos-5-x64`, then a user could also specify that value for `spec.imageName` as long as the following is true: -* There is no other `VirtualMachineImage` in the same namespace with that friendly name. -* There is no other `ClusterVirtualMachineImage` with the same friendly name. +* There is no other `VirtualMachineImage` in the same namespace with that display name. +* There is no other `ClusterVirtualMachineImage` with the same display name. -If the friendly name unambiguously resolves to the distinct, VM image `vmi-0a0044d7c690bcbea`, then a mutation webhook replaces `spec.imageName: photonos-5-x64` with `spec.imageName: vmi-0a0044d7c690bcbea`. If the friendly name resolves to multiple or no VM images, then the mutation webhook denies the request and outputs an error message accordingly. +If the display name unambiguously resolves to the distinct, VM image `vmi-0a0044d7c690bcbea`, then a mutation webhook replaces `spec.imageName: photonos-5-x64` with `spec.imageName: vmi-0a0044d7c690bcbea`. If the display name resolves to multiple or no VM images, then the mutation webhook denies the request and outputs an error message accordingly. ## Recommended Images diff --git a/docs/ref/api/v1alpha1.md b/docs/ref/api/v1alpha1.md index e41183c40..6c08cd871 100644 --- a/docs/ref/api/v1alpha1.md +++ b/docs/ref/api/v1alpha1.md @@ -457,6 +457,7 @@ _Appears in:_ | Field | Description | | --- | --- | | `Gateway4` _string_ | Gateway4 is the gateway for the IPv4 address family for this device. | +| `MacAddress` _string_ | MacAddress is the MAC address of the network device. | | `IPAddresses` _string array_ | IpAddresses represents one or more IP addresses assigned to the network device in CIDR notation, ex. "192.0.2.1/16". | ### NetworkInterfaceProviderReference @@ -676,6 +677,9 @@ _Appears in:_ | Field | Description | | --- | --- | +| `controllerName` _string_ | ControllerName describes the name of the controller responsible for reconciling VirtualMachine resources that are realized from this VirtualMachineClass. + When omitted, controllers reconciling VirtualMachine resources determine the default controller name from the environment variable DEFAULT_VM_CLASS_CONTROLLER_NAME. If this environment variable is not defined or empty, it defaults to vmoperator.vmware.com/vsphere. + Once a non-empty value is assigned to this field, attempts to set this field to an empty value will be silently ignored. | | `hardware` _[VirtualMachineClassHardware](#virtualmachineclasshardware)_ | Hardware describes the configuration of the VirtualMachineClass attributes related to virtual hardware. The configuration specified in this field is used to customize the virtual hardware characteristics of any VirtualMachine associated with this VirtualMachineClass. | | `policies` _[VirtualMachineClassPolicies](#virtualmachineclasspolicies)_ | Policies describes the configuration of the VirtualMachineClass attributes related to virtual infrastructure policy. The configuration specified in this field is used to customize various policies related to infrastructure resource consumption. | | `description` _string_ | Description describes the configuration of the VirtualMachineClass which is not related to virtual hardware or infrastructure policy. This field is used to address remaining specs about this VirtualMachineClass. | @@ -748,7 +752,7 @@ _Appears in:_ | `uuid` _string_ | Deprecated | | `internalId` _string_ | Deprecated | | `powerState` _string_ | Deprecated | -| `imageName` _string_ | ImageName describes the display name of this VirtualMachineImage. | +| `imageName` _string_ | ImageName describes the display name of this image. | | `imageSupported` _boolean_ | ImageSupported indicates whether the VirtualMachineImage is supported by VMService. A VirtualMachineImage is supported by VMService if the following conditions are true: - VirtualMachineImageV1Alpha1CompatibleCondition | | `conditions` _[Condition](#condition) array_ | Conditions describes the current condition information of the VirtualMachineImage object. e.g. if the OS type is supported or image is supported by VMService | | `contentLibraryRef` _[TypedLocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#typedlocalobjectreference-v1-core)_ | ContentLibraryRef is a reference to the source ContentLibrary/ClusterContentLibrary resource. | @@ -891,7 +895,7 @@ _Appears in:_ | Field | Description | | --- | --- | -| `name` _string_ | Name is the name of the published object. +| `name` _string_ | Name is the display name of the published object. If the spec.target.location.apiVersion equals imageregistry.vmware.com/v1alpha1 and the spec.target.location.kind equals ContentLibrary, then this should be the name that will show up in vCenter Content Library, not the custom resource name in the namespace. If omitted then the controller will use spec.source.name + "-image". | | `description` _string_ | Description is the description to assign to the published object. | @@ -1017,7 +1021,21 @@ _Appears in:_ | --- | --- | | `imageName` _string_ | ImageName describes the name of a VirtualMachineImage that is to be used as the base Operating System image of the desired VirtualMachine instances. The VirtualMachineImage resources can be introspected to discover identifying attributes that may help users to identify the desired image to use. | | `className` _string_ | ClassName describes the name of a VirtualMachineClass that is to be used as the overlaid resource configuration of VirtualMachine. A VirtualMachineClass is used to further customize the attributes of the VirtualMachine instance. See VirtualMachineClass for more description. | -| `powerState` _VirtualMachinePowerState_ | PowerState describes the desired power state of a VirtualMachine. Valid power states are "poweredOff" and "poweredOn". | +| `powerState` _VirtualMachinePowerState_ | PowerState describes the desired power state of a VirtualMachine. + Please note this field may be omitted when creating a new VM and will default to "poweredOn." However, once the field is set to a non-empty value, it may no longer be set to an empty value. + Additionally, setting this value to "suspended" is not supported when creating a new VM. The valid values when creating a new VM are "poweredOn" and "poweredOff." An empty value is also allowed on create since this value defaults to "poweredOn" for new VMs. | +| `powerOffMode` _VirtualMachinePowerOpMode_ | PowerOffMode describes the desired behavior when powering off a VM. + There are three, supported power off modes: hard, soft, and trySoft. The first mode, hard, is the equivalent of a physical system's power cord being ripped from the wall. The soft mode requires the VM's guest to have VM Tools installed and attempts to gracefully shutdown the VM. Its variant, trySoft, first attempts a graceful shutdown, and if that fails or the VM is not in a powered off state after five minutes, the VM is halted. + If omitted, the mode defaults to hard. | +| `suspendMode` _VirtualMachinePowerOpMode_ | SuspendMode describes the desired behavior when suspending a VM. + There are three, supported suspend modes: hard, soft, and trySoft. The first mode, hard, is where vSphere suspends the VM to disk without any interaction inside of the guest. The soft mode requires the VM's guest to have VM Tools installed and attempts to gracefully suspend the VM. Its variant, trySoft, first attempts a graceful suspend, and if that fails or the VM is not in a put into standby by the guest after five minutes, the VM is suspended. + If omitted, the mode defaults to hard. | +| `nextRestartTime` _string_ | NextRestartTime may be used to restart the VM, in accordance with RestartMode, by setting the value of this field to "now" (case-insensitive). + A mutating webhook changes this value to the current time (UTC), which the VM controller then uses to determine the VM should be restarted by comparing the value to the timestamp of the last time the VM was restarted. + Please note it is not possible to schedule future restarts using this field. The only value that users may set is the string "now" (case-insensitive). | +| `restartMode` _VirtualMachinePowerOpMode_ | RestartMode describes the desired behavior for restarting a VM when spec.nextRestartTime is set to "now" (case-insensitive). + There are three, supported suspend modes: hard, soft, and trySoft. The first mode, hard, is where vSphere resets the VM without any interaction inside of the guest. The soft mode requires the VM's guest to have VM Tools installed and asks the guest to restart the VM. Its variant, trySoft, first attempts a soft restart, and if that fails or does not complete within five minutes, the VM is hard reset. + If omitted, the mode defaults to hard. | | `ports` _[VirtualMachinePort](#virtualmachineport) array_ | Ports is currently unused and can be considered deprecated. | | `vmMetadata` _[VirtualMachineMetadata](#virtualmachinemetadata)_ | VmMetadata describes any optional metadata that should be passed to the Guest OS. | | `storageClass` _string_ | StorageClass describes the name of a StorageClass that should be used to configure storage-related attributes of the VirtualMachine instance. | @@ -1050,6 +1068,7 @@ _Appears in:_ | `changeBlockTracking` _boolean_ | ChangeBlockTracking describes the CBT enablement status on the VirtualMachine. | | `networkInterfaces` _[NetworkInterfaceStatus](#networkinterfacestatus) array_ | NetworkInterfaces describes a list of current status information for each network interface that is desired to be attached to the VirtualMachine. | | `zone` _string_ | Zone describes the availability zone where the VirtualMachine has been scheduled. Please note this field may be empty when the cluster is not zone-aware. | +| `lastRestartTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#time-v1-meta)_ | LastRestartTime describes the last time the VM was restarted. | ### VirtualMachineVolume diff --git a/docs/ref/api/v1alpha2.md b/docs/ref/api/v1alpha2.md index cf196b6f1..fadd339d9 100644 --- a/docs/ref/api/v1alpha2.md +++ b/docs/ref/api/v1alpha2.md @@ -220,6 +220,20 @@ _Appears in:_ | --- | --- | | `size` _Quantity_ | | +### InstanceVolumeClaimVolumeSource + + + +InstanceVolumeClaimVolumeSource contains information about the instance storage volume claimed as a PVC. + +_Appears in:_ +- [PersistentVolumeClaimVolumeSource](#persistentvolumeclaimvolumesource) + +| Field | Description | +| --- | --- | +| `storageClass` _string_ | StorageClass is the name of the Kubernetes StorageClass that provides the backing storage for this instance storage volume. | +| `size` _Quantity_ | Size is the size of the requested instance storage volume. | + ### LoadBalancerIngress @@ -259,6 +273,7 @@ _Appears in:_ | Field | Description | | --- | --- | | `Gateway4` _string_ | Gateway4 is the gateway for the IPv4 address family for this device. | +| `MacAddress` _string_ | MacAddress is the MAC address of the network device. | | `IPAddresses` _string array_ | IpAddresses represents one or more IP addresses assigned to the network device in CIDR notation, ex. "192.0.2.1/16". | ### NetworkStatus @@ -290,6 +305,21 @@ _Appears in:_ | `type` _string_ | Type describes the OVF property's type. | | `default` _string_ | Default describes the OVF property's default value. | +### PersistentVolumeClaimVolumeSource + + + +PersistentVolumeClaimVolumeSource is a composite for the Kubernetes corev1.PersistentVolumeClaimVolumeSource and instance storage options. + +_Appears in:_ +- [VirtualMachineVolumeSource](#virtualmachinevolumesource) + +| Field | Description | +| --- | --- | +| `claimName` _string_ | claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims | +| `readOnly` _boolean_ | readOnly Will force the ReadOnly setting in VolumeMounts. Default false. | +| `instanceVolumeClaim` _[InstanceVolumeClaimVolumeSource](#instancevolumeclaimvolumesource)_ | InstanceVolumeClaim is set if the PVC is backed by instance storage. | + ### ResourcePoolSpec @@ -374,7 +404,7 @@ _Appears in:_ | --- | --- | | `bootDiskCapacity` _Quantity_ | BootDiskCapacity is the capacity of the VM's boot disk -- the first disk from the VirtualMachineImage from which the VM was deployed. Please note it is not advised to change this value while the VM is running. Also, resizing the VM's boot disk may require actions inside of the guest to take advantage of the additional capacity. Finally, changing the size of the VM's boot disk, even increasing it, could adversely affect the VM. | -| `defaultVolumeProvisioningMode` _string_ | DefaultVolumeProvisioningMode specifies the default provisioning mode for persistent volumes managed by this VM. | +| `defaultVolumeProvisioningMode` _VirtualMachineVolumeProvisioningMode_ | DefaultVolumeProvisioningMode specifies the default provisioning mode for persistent volumes managed by this VM. | | `changeBlockTracking` _boolean_ | ChangeBlockTracking is a flag that enables incremental backup support for this VM, a feature utilized by external backup systems such as VMware Data Recovery. | ### VirtualMachineBootstrapCloudInitSpec @@ -399,7 +429,7 @@ _Appears in:_ -VirtualMachineBootstrapLinuxPrepSpec +VirtualMachineBootstrapLinuxPrepSpec describes the LinuxPrep configuration used to bootstrap the VM. _Appears in:_ - [VirtualMachineBootstrapSpec](#virtualmachinebootstrapspec) @@ -463,7 +493,7 @@ _Appears in:_ -VirtualMachineBootstrapVAppConfigSpec +VirtualMachineBootstrapVAppConfigSpec describes the vApp configuration used to bootstrap the VM. _Appears in:_ - [VirtualMachineBootstrapSpec](#virtualmachinebootstrapspec) @@ -529,6 +559,9 @@ _Appears in:_ | Field | Description | | --- | --- | +| `controllerName` _string_ | ControllerName describes the name of the controller responsible for reconciling VirtualMachine resources that are realized from this VirtualMachineClass. + When omitted, controllers reconciling VirtualMachine resources determine the default controller name from the environment variable DEFAULT_VM_CLASS_CONTROLLER_NAME. If this environment variable is not defined or empty, it defaults to vmoperator.vmware.com/vsphere. + Once a non-empty value is assigned to this field, attempts to set this field to an empty value will be silently ignored. | | `hardware` _[VirtualMachineClassHardware](#virtualmachineclasshardware)_ | Hardware describes the configuration of the VirtualMachineClass attributes related to virtual hardware. The configuration specified in this field is used to customize the virtual hardware characteristics of any VirtualMachine associated with this VirtualMachineClass. | | `policies` _[VirtualMachineClassPolicies](#virtualmachineclasspolicies)_ | Policies describes the configuration of the VirtualMachineClass attributes related to virtual infrastructure policy. The configuration specified in this field is used to customize various policies related to infrastructure resource consumption. | | `description` _string_ | Description describes the configuration of the VirtualMachineClass which is not related to virtual hardware or infrastructure policy. This field is used to address remaining specs about this VirtualMachineClass. | @@ -615,7 +648,7 @@ _Appears in:_ | Field | Description | | --- | --- | -| `name` _string_ | Name describes the observed, "friendly" name for this image. | +| `name` _string_ | Name describes the display name of this image. | | `capabilities` _string array_ | Capabilities describes the image's observed capabilities. The capabilities are discerned when VM Operator reconciles an image. If the source of an image is an OVF in Content Library, then the capabilities are parsed from the OVF property capabilities.image.vmoperator.vmware.com as a comma-separated list of values. Well-known capabilities include: * cloud-init * nvidia-gpu * sriov-net @@ -626,6 +659,8 @@ _Appears in:_ The OS information is also added to the image resource's labels. Please refer to VirtualMachineImageOSInfo for more information. | | `ovfProperties` _[OVFProperty](#ovfproperty) array_ | OVFProperties describes the observed OVF properties defined for this image. | | `productInfo` _[VirtualMachineImageProductInfo](#virtualmachineimageproductinfo)_ | ProductInfo describes the observed product information for this image. | +| `providerContentVersion` _string_ | ProviderContentVersion describes the content version from the provider item that this image corresponds to. If the provider of this image is a Content Library, this will be the version of the corresponding Content Library item. | +| `providerItemID` _string_ | ProviderItemID describes the ID of the provider item that this image corresponds to. If the provider of this image is a Content Library, this ID will be that of the corresponding Content Library item. | | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#condition-v1-meta) array_ | Conditions describes the observed conditions for this image. | ### VirtualMachineNetworkDHCPOptionsStatus @@ -713,7 +748,7 @@ _Appears in:_ -VirtualMachineNetworkIPStackStatus describes the observed state of a a VM's IP stack. +VirtualMachineNetworkIPStackStatus describes the observed state of a VM's IP stack. _Appears in:_ - [VirtualMachineNetworkStatus](#virtualmachinenetworkstatus) @@ -801,12 +836,12 @@ _Appears in:_ | `mtu` _integer_ | MTU is the Maximum Transmission Unit size in bytes. Please note this feature is available only with the following bootstrap providers: CloudInit. | | `nameservers` _string array_ | Nameservers is a list of IP4 and/or IP6 addresses used as DNS nameservers. - Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep. + Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). Please note that Linux allows only three nameservers (https://linux.die.net/man/5/resolv.conf). | | `routes` _[VirtualMachineNetworkRouteSpec](#virtualmachinenetworkroutespec) array_ | Routes is a list of optional, static routes. Please note this feature is available only with the following bootstrap providers: CloudInit. | | `searchDomains` _string array_ | SearchDomains is a list of search domains used when resolving IP addresses with DNS. - Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep. | + Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). | ### VirtualMachineNetworkInterfaceStatus @@ -836,8 +871,8 @@ _Appears in:_ | Field | Description | | --- | --- | -| `to` _string_ | To is an IP4 address. | -| `via` _string_ | Via is an IP4 address. | +| `to` _string_ | To is an IP4 or IP6 address. | +| `via` _string_ | Via is an IP4 or IP6 address. | | `metric` _integer_ | Metric is the weight/priority of the route. | @@ -858,7 +893,7 @@ _Appears in:_ | `disabled` _boolean_ | Disabled is a flag that indicates whether or not to disable networking for this VM. When set to true, the VM is not configured with a default interface nor any specified from the Interfaces field. | | `hostName` _string_ | HostName is the value the guest uses as its host name. If omitted then the name of the VM will be used. - Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep. | + Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). | | `interfaces` _[VirtualMachineNetworkInterfaceSpec](#virtualmachinenetworkinterfacespec) array_ | Interfaces is the list of network interfaces used by this VM. Please note this field is mutually exclusive with the following fields: DeviceName, Network, Addresses, DHCP4, DHCP6, Gateway4, Gateway6, MTU, Nameservers, Routes, and SearchDomains. | | `deviceName` _string_ | DeviceName describes the unique name of this network interface, used to distinguish it from other network interfaces attached to this VM. @@ -881,27 +916,25 @@ _Appears in:_ | `gateway4` _string_ | Gateway4 is the default, IP4 gateway for this VM. Please note this field is only supported if the network connection supports manual IP allocation. If the network connection supports manual IP allocation and the Addresses field includes at least one IP4 address, then this field is required. - Please note the IP address must include the network prefix length, ex. 192.168.0.1/24. Please note this field is mutually exclusive with DHCP4. Please note if the Interfaces field is non-empty then this field is ignored and should be specified on the elements in the Interfaces list. | | `gateway6` _string_ | Gateway6 is the primary IP6 gateway for this VM. Please note this field is only supported if the network connection supports manual IP allocation. If the network connection supports manual IP allocation and the Addresses field includes at least one IP4 address, then this field is required. - Please note the IP address must include the network prefix length, ex. 2001:db8:101::1/64. Please note this field is mutually exclusive with DHCP6. Please note if the Interfaces field is non-empty then this field is ignored and should be specified on the elements in the Interfaces list. | | `mtu` _integer_ | MTU is the Maximum Transmission Unit size in bytes. Please note this feature is available only with the following bootstrap providers: CloudInit. Please note if the Interfaces field is non-empty then this field is ignored and should be specified on the elements in the Interfaces list. | | `nameservers` _string array_ | Nameservers is a list of IP4 and/or IP6 addresses used as DNS nameservers. - Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep. + Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). Please note that Linux allows only three nameservers (https://linux.die.net/man/5/resolv.conf). Please note if the Interfaces field is non-empty then this field is ignored and should be specified on the elements in the Interfaces list. | | `routes` _[VirtualMachineNetworkRouteSpec](#virtualmachinenetworkroutespec) array_ | Routes is a list of optional, static routes. Please note this feature is available only with the following bootstrap providers: CloudInit. Please note if the Interfaces field is non-empty then this field is ignored and should be specified on the elements in the Interfaces list. | | `searchDomains` _string array_ | SearchDomains is a list of search domains used when resolving IP addresses with DNS. - Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep. + Please note this feature is available only with the following bootstrap providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). Please note if the Interfaces field is non-empty then this field is ignored and should be specified on the elements in the Interfaces list. | ### VirtualMachineNetworkStatus @@ -917,11 +950,11 @@ _Appears in:_ | --- | --- | | `interfaces` _[VirtualMachineNetworkInterfaceStatus](#virtualmachinenetworkinterfacestatus) array_ | Interfaces describes the status of the VM's network interfaces. | | `primaryIP4` _string_ | PrimaryIP4 describes the VM's primary IP4 address. - If the bootstrap provider is CloudInit then this value is set to the value of the VM's "guestinfo.local-ipv4" property. Please see https://bit.ly/3A66vZg for more information on how this value is calculated. - If the bootstrap provider is anything else then this field is set to the the value of the infrastructure VM's "guest.ipAddress" field. Please see https://bit.ly/3Au0jM4 for more information. | + If the bootstrap provider is CloudInit then this value is set to the value of the VM's "guestinfo.local-ipv4" property. Please see https://bit.ly/3NJB534 for more information on how this value is calculated. + If the bootstrap provider is anything else then this field is set to the value of the infrastructure VM's "guest.ipAddress" field. Please see https://bit.ly/3Au0jM4 for more information. | | `primaryIP6` _string_ | PrimaryIP6 describes the VM's primary IP6 address. - If the bootstrap provider is CloudInit then this value is set to the value of the VM's "guestinfo.local-ipv6" property. Please see https://bit.ly/3A66vZg for more information on how this value is calculated. - If the bootstrap provider is anything else then this field is set to the the value of the infrastructure VM's "guest.ipAddress" field. Please see https://bit.ly/3Au0jM4 for more information. | + If the bootstrap provider is CloudInit then this value is set to the value of the VM's "guestinfo.local-ipv6" property. Please see https://bit.ly/3NJB534 for more information on how this value is calculated. + If the bootstrap provider is anything else then this field is set to the value of the infrastructure VM's "guest.ipAddress" field. Please see https://bit.ly/3Au0jM4 for more information. | | `dhcp` _[VirtualMachineNetworkDHCPStatus](#virtualmachinenetworkdhcpstatus)_ | DHCP describes the VM's observed, client-side, system-wide DHCP options. | | `dns` _[VirtualMachineNetworkDNSStatus](#virtualmachinenetworkdnsstatus)_ | DNS describes the VM's observed, client-side DNS configuration. | | `ipRoutes` _[VirtualMachineNetworkIPRouteStatus](#virtualmachinenetworkiproutestatus) array_ | IPRoutes contain the VM's routing tables for all address families. | @@ -1204,7 +1237,21 @@ _Appears in:_ Please note that defaulting to Sysprep for Windows images only works if the image uses a volume license key, otherwise the image's product ID is required. | | `network` _[VirtualMachineNetworkSpec](#virtualmachinenetworkspec)_ | Network describes the desired network configuration for the VM. Please note this value may be omitted entirely and the VM will be assigned a single, virtual network interface that is connected to the Namespace's default network. | -| `powerState` _VirtualMachinePowerState_ | PowerState describes the desired power state of a VirtualMachine. | +| `powerState` _VirtualMachinePowerState_ | PowerState describes the desired power state of a VirtualMachine. + Please note this field may be omitted when creating a new VM and will default to "PoweredOn." However, once the field is set to a non-empty value, it may no longer be set to an empty value. + Additionally, setting this value to "Suspended" is not supported when creating a new VM. The valid values when creating a new VM are "PoweredOn" and "PoweredOff." An empty value is also allowed on create since this value defaults to "PoweredOn" for new VMs. | +| `powerOffMode` _VirtualMachinePowerOpMode_ | PowerOffMode describes the desired behavior when powering off a VM. + There are three, supported power off modes: Hard, Soft, and TrySoft. The first mode, Hard, is the equivalent of a physical system's power cord being ripped from the wall. The Soft mode requires the VM's guest to have VM Tools installed and attempts to gracefully shutdown the VM. Its variant, TrySoft, first attempts a graceful shutdown, and if that fails or the VM is not in a powered off state after five minutes, the VM is halted. + If omitted, the mode defaults to TrySoft. | +| `suspendMode` _VirtualMachinePowerOpMode_ | SuspendMode describes the desired behavior when suspending a VM. + There are three, supported suspend modes: Hard, Soft, and TrySoft. The first mode, Hard, is where vSphere suspends the VM to disk without any interaction inside of the guest. The Soft mode requires the VM's guest to have VM Tools installed and attempts to gracefully suspend the VM. Its variant, TrySoft, first attempts a graceful suspend, and if that fails or the VM is not in a put into standby by the guest after five minutes, the VM is suspended. + If omitted, the mode defaults to TrySoft. | +| `nextRestartTime` _string_ | NextRestartTime may be used to restart the VM, in accordance with RestartMode, by setting the value of this field to "now" (case-insensitive). + A mutating webhook changes this value to the current time (UTC), which the VM controller then uses to determine the VM should be restarted by comparing the value to the timestamp of the last time the VM was restarted. + Please note it is not possible to schedule future restarts using this field. The only value that users may set is the string "now" (case-insensitive). | +| `restartMode` _VirtualMachinePowerOpMode_ | RestartMode describes the desired behavior for restarting a VM when spec.nextRestartTime is set to "now" (case-insensitive). + There are three, supported suspend modes: Hard, Soft, and TrySoft. The first mode, Hard, is where vSphere resets the VM without any interaction inside of the guest. The Soft mode requires the VM's guest to have VM Tools installed and asks the guest to restart the VM. Its variant, TrySoft, first attempts a soft restart, and if that fails or does not complete within five minutes, the VM is hard reset. + If omitted, the mode defaults to TrySoft. | | `volumes` _[VirtualMachineVolume](#virtualmachinevolume) array_ | Volumes describes a list of volumes that can be mounted to the VM. | | `readinessProbe` _[VirtualMachineReadinessProbeSpec](#virtualmachinereadinessprobespec)_ | ReadinessProbe describes a probe used to determine the VM's ready state. | | `readinessGates` _[VirtualMachineReadinessGate](#virtualmachinereadinessgate) array_ | ReadinessGates, if specified, will be evaluated to determine the VM's readiness. @@ -1237,6 +1284,7 @@ _Appears in:_ | `changeBlockTracking` _boolean_ | ChangeBlockTracking describes the CBT enablement status on the VM. | | `zone` _string_ | Zone describes the availability zone where the VirtualMachine has been scheduled. Please note this field may be empty when the cluster is not zone-aware. | +| `lastRestartTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#time-v1-meta)_ | LastRestartTime describes the last time the VM was restarted. | ### VirtualMachineVolume @@ -1251,10 +1299,9 @@ _Appears in:_ | Field | Description | | --- | --- | | `name` _string_ | Name represents the volume's name. Must be a DNS_LABEL and unique within the VM. | -| `persistentVolumeClaim` _[PersistentVolumeClaimVolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaimvolumesource-v1-core)_ | PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. +| `persistentVolumeClaim` _[PersistentVolumeClaimVolumeSource](#persistentvolumeclaimvolumesource)_ | PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. More information is available at https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims. | - ### VirtualMachineVolumeSource @@ -1266,7 +1313,7 @@ _Appears in:_ | Field | Description | | --- | --- | -| `persistentVolumeClaim` _[PersistentVolumeClaimVolumeSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#persistentvolumeclaimvolumesource-v1-core)_ | PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. +| `persistentVolumeClaim` _[PersistentVolumeClaimVolumeSource](#persistentvolumeclaimvolumesource)_ | PersistentVolumeClaim represents a reference to a PersistentVolumeClaim in the same namespace. More information is available at https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims. | ### VirtualMachineVolumeStatus @@ -1297,6 +1344,7 @@ _Appears in:_ | Field | Description | | --- | --- | | `name` _string_ | Name is the name of a VM in the same Namespace as this web console request. | +| `publicKey` _string_ | PublicKey is used to encrypt the status.response. This is expected to be a RSA OAEP public key in X.509 PEM format. | ### VirtualMachineWebConsoleRequestStatus @@ -1309,6 +1357,7 @@ _Appears in:_ | Field | Description | | --- | --- | +| `response` _string_ | Response will be the authenticated ticket corresponding to this web console request. | | `expiryTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#time-v1-meta)_ | ExpiryTime is the time at which access via this request will expire. | | `proxyAddr` _string_ | ProxyAddr describes the host address and optional port used to access the VM's web console. The value could be a DNS entry, IPv4, or IPv6 address, followed by an optional port. For example, valid values include: From 30c3741f187b24eae9800ae4f14218ed91d1efdd Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Tue, 17 Oct 2023 18:30:44 -0700 Subject: [PATCH 31/54] Fix deploy-local script for v1a2 disabled scenarios (#250) set v1a1 as the storage version, remove v1a2 CRDs and conversion webhooks. --- hack/deploy-local.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/hack/deploy-local.sh b/hack/deploy-local.sh index c52025043..8d68f4380 100755 --- a/hack/deploy-local.sh +++ b/hack/deploy-local.sh @@ -53,10 +53,19 @@ fi FSS_V1A2=$(yq '.spec.template.spec.containers[]| select(.name == "manager") | .env[] | select(.name == "FSS_WCP_VMSERVICE_V1ALPHA2") | .value' "$YAML") if [ "${FSS_V1A2}" = "false" ]; then \ - yq -i 'del(.spec.versions[] | select(.name == "v1alpha2"))' "$YAML" + # remove conversion webhooks from CRDs + yq -i 'del(select(.kind == "CustomResourceDefinition") | .spec.conversion | select(.strategy == "Webhook"))' "$YAML" + # set storage version to v1alpha1 + yq -i '(select(.kind == "CustomResourceDefinition") | .spec.versions[] | select(.name == "v1alpha1")).storage = true' "$YAML" + # remove v1alpha2 versions from CRDs + yq -i 'del(select(.kind == "CustomResourceDefinition") | .spec.versions[] | select(.name == "v1alpha2"))' "$YAML" + # remove CRDs with empty spec.versions + yq -i 'del(select(.kind == "CustomResourceDefinition") | select(.spec.versions | length == 0))' "$YAML" + # remove all v1alpha2 webhooks yq -i 'del(.webhooks[] | select(.name == "*v1alpha2*"))' "$YAML" fi; + $KUBECTL apply -f "$YAML" if [[ -n $DEPLOYMENT_EXISTS ]]; then From c7be07a5f14e4f9854dccfec8e921df27f62d236 Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Wed, 18 Oct 2023 16:20:24 -0400 Subject: [PATCH 32/54] =?UTF-8?q?=E2=9C=A8=20Update=20VM=20backup=20to=20s?= =?UTF-8?q?kip=20unchanged=20data=20and=20include=20instance=20ID=20(#239)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch includes the following changes for the VM backup & restore workflow: - Move ExtraConfigToMap and MergeExtraConfig util functions to "pkg/util/configspec.go" - Persist cloud-init instance ID to keep it unchanged upon VM restore. - Optimize backup process to skip reconfiguring the VM with unchanged data. - Persist backup info in VM's EC after successfully updating the VM object to avoid additional calls from the controller. --- .../v1alpha1/virtualmachine_controller.go | 9 - .../virtualmachine_controller_unit_test.go | 23 --- pkg/util/configspec.go | 27 +++ pkg/util/configspec_test.go | 77 ++++++++ pkg/vmprovider/fake/fake_vm_provider.go | 12 -- pkg/vmprovider/interface.go | 1 - .../providers/vsphere/constants/constants.go | 3 + .../providers/vsphere/session/session_util.go | 26 --- .../vsphere/session/session_util_test.go | 77 -------- .../session/session_vm_customization.go | 4 +- .../session/session_vm_customization_test.go | 14 +- .../vsphere/session/session_vm_update.go | 8 +- .../vsphere/virtualmachine/backup.go | 174 ++++++++++++++---- .../vsphere/virtualmachine/backup_test.go | 137 ++++++++++++-- .../providers/vsphere/vmprovider_vm.go | 38 ++-- .../vsphere2/session/session_util.go | 34 ---- .../vsphere2/session/session_util_test.go | 93 ---------- .../vsphere2/session/session_vm_update.go | 8 +- .../vmlifecycle/bootstrap_cloudinit_test.go | 25 +-- 19 files changed, 403 insertions(+), 387 deletions(-) delete mode 100644 pkg/vmprovider/providers/vsphere2/session/session_util.go delete mode 100644 pkg/vmprovider/providers/vsphere2/session/session_util_test.go diff --git a/controllers/virtualmachine/v1alpha1/virtualmachine_controller.go b/controllers/virtualmachine/v1alpha1/virtualmachine_controller.go index 1105d0d28..a91f95929 100644 --- a/controllers/virtualmachine/v1alpha1/virtualmachine_controller.go +++ b/controllers/virtualmachine/v1alpha1/virtualmachine_controller.go @@ -471,15 +471,6 @@ func (r *Reconciler) ReconcileNormal(ctx *context.VirtualMachineContext) (reterr // Add this VM to prober manager if ReconcileNormal succeeds. r.Prober.AddToProberManager(ctx.VM) - // Back up this VM if ReconcileNormal succeeds and the FSS is enabled. - if lib.IsVMServiceBackupRestoreFSSEnabled() { - if err := r.VMProvider.BackupVirtualMachine(ctx, ctx.VM); err != nil { - ctx.Logger.Error(err, "Failed to backup VirtualMachine") - r.Recorder.EmitEvent(ctx.VM, "Backup", err, false) - return err - } - } - ctx.Logger.Info("Finished Reconciling VirtualMachine") return nil } diff --git a/controllers/virtualmachine/v1alpha1/virtualmachine_controller_unit_test.go b/controllers/virtualmachine/v1alpha1/virtualmachine_controller_unit_test.go index 876870fa3..2777d593b 100644 --- a/controllers/virtualmachine/v1alpha1/virtualmachine_controller_unit_test.go +++ b/controllers/virtualmachine/v1alpha1/virtualmachine_controller_unit_test.go @@ -6,7 +6,6 @@ package v1alpha1_test import ( "context" "errors" - "os" "strings" . "github.com/onsi/ginkgo" @@ -19,7 +18,6 @@ import ( virtualmachine "github.com/vmware-tanzu/vm-operator/controllers/virtualmachine/v1alpha1" vmopContext "github.com/vmware-tanzu/vm-operator/pkg/context" - "github.com/vmware-tanzu/vm-operator/pkg/lib" proberfake "github.com/vmware-tanzu/vm-operator/pkg/prober/fake" providerfake "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/fake" "github.com/vmware-tanzu/vm-operator/test/builder" @@ -156,27 +154,6 @@ func unitTestsReconcile() { Expect(reconciler.ReconcileNormal(vmCtx)).Should(Succeed()) Expect(fakeProbeManager.IsAddToProberManagerCalled).Should(BeTrue()) }) - - When("The VM Service Backup and Restore FSS is enabled", func() { - BeforeEach(func() { - Expect(os.Setenv(lib.VMServiceBackupRestoreFSS, lib.TrueString)).To(Succeed()) - }) - - AfterEach(func() { - Expect(os.Unsetenv(lib.VMServiceBackupRestoreFSS)).To(Succeed()) - }) - - It("Should call backup Virtual Machine if ReconcileNormal succeeds", func() { - var isBackupVirtualMachineCalled bool - fakeVMProvider.BackupVirtualMachineFn = func(ctx context.Context, vm *vmopv1.VirtualMachine) error { - isBackupVirtualMachineCalled = true - return nil - } - - Expect(reconciler.ReconcileNormal(vmCtx)).Should(Succeed()) - Expect(isBackupVirtualMachineCalled).Should(BeTrue()) - }) - }) }) Context("ReconcileDelete", func() { diff --git a/pkg/util/configspec.go b/pkg/util/configspec.go index 97d07d444..8a62d364b 100644 --- a/pkg/util/configspec.go +++ b/pkg/util/configspec.go @@ -188,3 +188,30 @@ func AppendNewExtraConfigValues( return append(extraConfig, newExtraConfig...) } + +// ExtraConfigToMap converts the ExtraConfig to a map with string values. +func ExtraConfigToMap(input []vimTypes.BaseOptionValue) (output map[string]string) { + output = make(map[string]string) + for _, opt := range input { + if optValue := opt.GetOptionValue(); optValue != nil { + // Only set string type values + if val, ok := optValue.Value.(string); ok { + output[optValue.Key] = val + } + } + } + return +} + +// MergeExtraConfig adds the key/value to the ExtraConfig if the key is not present. +// It returns the newly added ExtraConfig. +func MergeExtraConfig(extraConfig []vimTypes.BaseOptionValue, newMap map[string]string) []vimTypes.BaseOptionValue { + merged := make([]vimTypes.BaseOptionValue, 0) + ecMap := ExtraConfigToMap(extraConfig) + for k, v := range newMap { + if _, exists := ecMap[k]; !exists { + merged = append(merged, &vimTypes.OptionValue{Key: k, Value: v}) + } + } + return merged +} diff --git a/pkg/util/configspec_test.go b/pkg/util/configspec_test.go index f53594152..9d895f53c 100644 --- a/pkg/util/configspec_test.go +++ b/pkg/util/configspec_test.go @@ -386,6 +386,83 @@ var _ = Describe("SanitizeVMClassConfigSpec", func() { }) }) +var _ = Describe("ExtraConfigToMap", func() { + var ( + extraConfig []vimTypes.BaseOptionValue + extraConfigMap map[string]string + ) + BeforeEach(func() { + extraConfig = []vimTypes.BaseOptionValue{} + }) + JustBeforeEach(func() { + extraConfigMap = util.ExtraConfigToMap(extraConfig) + }) + + Context("Empty extraConfig", func() { + It("Return empty map", func() { + Expect(extraConfigMap).To(HaveLen(0)) + }) + }) + + Context("With extraConfig", func() { + BeforeEach(func() { + extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key1", Value: "value1"}) + extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key2", Value: "value2"}) + }) + It("Return valid map", func() { + Expect(extraConfigMap).To(HaveLen(2)) + Expect(extraConfigMap["key1"]).To(Equal("value1")) + Expect(extraConfigMap["key2"]).To(Equal("value2")) + }) + }) +}) + +var _ = Describe("MergeExtraConfig", func() { + var ( + extraConfig []vimTypes.BaseOptionValue + newMap map[string]string + merged []vimTypes.BaseOptionValue + ) + BeforeEach(func() { + extraConfig = []vimTypes.BaseOptionValue{ + &vimTypes.OptionValue{Key: "existingkey1", Value: "existingvalue1"}, + &vimTypes.OptionValue{Key: "existingkey2", Value: "existingvalue2"}, + } + newMap = map[string]string{} + }) + JustBeforeEach(func() { + merged = util.MergeExtraConfig(extraConfig, newMap) + }) + + Context("Empty newMap", func() { + It("Return empty merged", func() { + Expect(merged).To(BeEmpty()) + }) + }) + + Context("NewMap with existing key", func() { + BeforeEach(func() { + newMap["existingkey1"] = "existingkey1" + }) + It("Return empty merged", func() { + Expect(merged).To(BeEmpty()) + }) + }) + + Context("NewMap with new keys", func() { + BeforeEach(func() { + newMap["newkey1"] = "newvalue1" + newMap["newkey2"] = "newvalue2" + }) + It("Return merged map", func() { + Expect(merged).To(HaveLen(2)) + mergedMap := util.ExtraConfigToMap(merged) + Expect(mergedMap["newkey1"]).To(Equal("newvalue1")) + Expect(mergedMap["newkey2"]).To(Equal("newvalue2")) + }) + }) +}) + func mustParseTime(layout, value string) time.Time { t, err := time.Parse(layout, value) if err != nil { diff --git a/pkg/vmprovider/fake/fake_vm_provider.go b/pkg/vmprovider/fake/fake_vm_provider.go index 9079bb769..0be8779bd 100644 --- a/pkg/vmprovider/fake/fake_vm_provider.go +++ b/pkg/vmprovider/fake/fake_vm_provider.go @@ -31,7 +31,6 @@ type funcs struct { DeleteVirtualMachineFn func(ctx context.Context, vm *vmopv1.VirtualMachine) error PublishVirtualMachineFn func(ctx context.Context, vm *vmopv1.VirtualMachine, vmPub *vmopv1.VirtualMachinePublishRequest, cl *imgregv1a1.ContentLibrary, actID string) (string, error) - BackupVirtualMachineFn func(ctx context.Context, vm *vmopv1.VirtualMachine) error GetVirtualMachineGuestHeartbeatFn func(ctx context.Context, vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) GetVirtualMachineWebMKSTicketFn func(ctx context.Context, vm *vmopv1.VirtualMachine, pubKey string) (string, error) GetVirtualMachineHardwareVersionFn func(ctx context.Context, vm *vmopv1.VirtualMachine) (int32, error) @@ -113,17 +112,6 @@ func (s *VMProvider) PublishVirtualMachine(ctx context.Context, vm *vmopv1.Virtu return "dummy-id", nil } -func (s *VMProvider) BackupVirtualMachine(ctx context.Context, vm *vmopv1.VirtualMachine) error { - s.Lock() - defer s.Unlock() - - if s.BackupVirtualMachineFn != nil { - return s.BackupVirtualMachineFn(ctx, vm) - } - - return nil -} - func (s *VMProvider) GetVirtualMachineGuestHeartbeat(ctx context.Context, vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { s.Lock() defer s.Unlock() diff --git a/pkg/vmprovider/interface.go b/pkg/vmprovider/interface.go index b69c12b66..d8df9d25a 100644 --- a/pkg/vmprovider/interface.go +++ b/pkg/vmprovider/interface.go @@ -22,7 +22,6 @@ type VirtualMachineProviderInterface interface { DeleteVirtualMachine(ctx context.Context, vm *vmopv1.VirtualMachine) error PublishVirtualMachine(ctx context.Context, vm *vmopv1.VirtualMachine, vmPub *vmopv1.VirtualMachinePublishRequest, cl *imgregv1a1.ContentLibrary, actID string) (string, error) - BackupVirtualMachine(ctx context.Context, vm *vmopv1.VirtualMachine) error GetVirtualMachineGuestHeartbeat(ctx context.Context, vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) GetVirtualMachineWebMKSTicket(ctx context.Context, vm *vmopv1.VirtualMachine, pubKey string) (string, error) GetVirtualMachineHardwareVersion(ctx context.Context, vm *vmopv1.VirtualMachine) (int32, error) diff --git a/pkg/vmprovider/providers/vsphere/constants/constants.go b/pkg/vmprovider/providers/vsphere/constants/constants.go index be7b042cf..838d7cc33 100644 --- a/pkg/vmprovider/providers/vsphere/constants/constants.go +++ b/pkg/vmprovider/providers/vsphere/constants/constants.go @@ -128,4 +128,7 @@ const ( // BackupVMDiskDataExtraConfigKey is the ExtraConfig key to the VM's disk info // data in JSON, compressed using gzip and base64-encoded. BackupVMDiskDataExtraConfigKey = "vmservice.virtualmachine.diskdata" + // BackupVMCloudInitInstanceIDExtraConfigKey is the ExtraConfig key to the VM's + // Cloud-Init instance ID, compressed using gzip and base64-encoded. + BackupVMCloudInitInstanceIDExtraConfigKey = "vmservice.virtualmachine.cloudinit.instanceid" ) diff --git a/pkg/vmprovider/providers/vsphere/session/session_util.go b/pkg/vmprovider/providers/vsphere/session/session_util.go index bc46e66e4..7937c3ab6 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_util.go +++ b/pkg/vmprovider/providers/vsphere/session/session_util.go @@ -13,32 +13,6 @@ const ( OvfEnvironmentTransportGuestInfo = "com.vmware.guestInfo" ) -func ExtraConfigToMap(input []vimTypes.BaseOptionValue) (output map[string]string) { - output = make(map[string]string) - for _, opt := range input { - if optValue := opt.GetOptionValue(); optValue != nil { - // Only set string type values - if val, ok := optValue.Value.(string); ok { - output[optValue.Key] = val - } - } - } - return -} - -// MergeExtraConfig adds the key/value to the ExtraConfig if the key is not present, to let to the value be -// changed by the VM. The existing usage of ExtraConfig is hard to fit in the reconciliation model. -func MergeExtraConfig(extraConfig []vimTypes.BaseOptionValue, newMap map[string]string) []vimTypes.BaseOptionValue { - merged := make([]vimTypes.BaseOptionValue, 0) - ecMap := ExtraConfigToMap(extraConfig) - for k, v := range newMap { - if _, exists := ecMap[k]; !exists { - merged = append(merged, &vimTypes.OptionValue{Key: k, Value: v}) - } - } - return merged -} - // GetMergedvAppConfigSpec prepares a vApp VmConfigSpec which will set the vmMetadata supplied key/value fields. Only // fields marked userConfigurable and pre-existing on the VM (ie. originated from the OVF Image) // will be set, and all others will be ignored. diff --git a/pkg/vmprovider/providers/vsphere/session/session_util_test.go b/pkg/vmprovider/providers/vsphere/session/session_util_test.go index 773bb10e8..7328d9e1d 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_util_test.go +++ b/pkg/vmprovider/providers/vsphere/session/session_util_test.go @@ -79,81 +79,4 @@ var _ = Describe("Test Session Utils", func() { ), ) }) - - Context("ExtraConfigToMap", func() { - var ( - extraConfig []vimTypes.BaseOptionValue - extraConfigMap map[string]string - ) - BeforeEach(func() { - extraConfig = []vimTypes.BaseOptionValue{} - }) - JustBeforeEach(func() { - extraConfigMap = session.ExtraConfigToMap(extraConfig) - }) - - Context("Empty extraConfig", func() { - It("Return empty map", func() { - Expect(extraConfigMap).To(HaveLen(0)) - }) - }) - - Context("With extraConfig", func() { - BeforeEach(func() { - extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key1", Value: "value1"}) - extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key2", Value: "value2"}) - }) - It("Return valid map", func() { - Expect(extraConfigMap).To(HaveLen(2)) - Expect(extraConfigMap["key1"]).To(Equal("value1")) - Expect(extraConfigMap["key2"]).To(Equal("value2")) - }) - }) - }) - - Context("MergeExtraConfig", func() { - var ( - extraConfig []vimTypes.BaseOptionValue - newMap map[string]string - merged []vimTypes.BaseOptionValue - ) - BeforeEach(func() { - extraConfig = []vimTypes.BaseOptionValue{ - &vimTypes.OptionValue{Key: "existingkey1", Value: "existingvalue1"}, - &vimTypes.OptionValue{Key: "existingkey2", Value: "existingvalue2"}, - } - newMap = map[string]string{} - }) - JustBeforeEach(func() { - merged = session.MergeExtraConfig(extraConfig, newMap) - }) - - Context("Empty newMap", func() { - It("Return empty merged", func() { - Expect(merged).To(BeEmpty()) - }) - }) - - Context("NewMap with existing key", func() { - BeforeEach(func() { - newMap["existingkey1"] = "existingkey1" - }) - It("Return empty merged", func() { - Expect(merged).To(BeEmpty()) - }) - }) - - Context("NewMap with new keys", func() { - BeforeEach(func() { - newMap["newkey1"] = "newvalue1" - newMap["newkey2"] = "newvalue2" - }) - It("Return merged map", func() { - Expect(merged).To(HaveLen(2)) - mergedMap := session.ExtraConfigToMap(merged) - Expect(mergedMap["newkey1"]).To(Equal("newvalue1")) - Expect(mergedMap["newkey2"]).To(Equal("newvalue2")) - }) - }) - }) }) diff --git a/pkg/vmprovider/providers/vsphere/session/session_vm_customization.go b/pkg/vmprovider/providers/vsphere/session/session_vm_customization.go index 6e6e9ba6e..1089cb281 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_vm_customization.go +++ b/pkg/vmprovider/providers/vsphere/session/session_vm_customization.go @@ -192,7 +192,7 @@ func GetCloudInitGuestInfoCustSpec( } configSpec := &vimTypes.VirtualMachineConfigSpec{} - configSpec.ExtraConfig = MergeExtraConfig(config.ExtraConfig, extraConfig) + configSpec.ExtraConfig = util.MergeExtraConfig(config.ExtraConfig, extraConfig) // Remove the VAppConfig to ensure Cloud-Init inside of the guest does not // activate and prefer the OVF datasource over the VMware datasource. @@ -217,7 +217,7 @@ func GetExtraConfigCustSpec( } configSpec := &vimTypes.VirtualMachineConfigSpec{} - configSpec.ExtraConfig = MergeExtraConfig(config.ExtraConfig, extraConfig) + configSpec.ExtraConfig = util.MergeExtraConfig(config.ExtraConfig, extraConfig) return configSpec } diff --git a/pkg/vmprovider/providers/vsphere/session/session_vm_customization_test.go b/pkg/vmprovider/providers/vsphere/session/session_vm_customization_test.go index d56e7ecff..ae584d620 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_vm_customization_test.go +++ b/pkg/vmprovider/providers/vsphere/session/session_vm_customization_test.go @@ -150,7 +150,7 @@ var _ = Describe("Customization via ConfigSpec", func() { It("Updates configSpec.ExtraConfig", func() { Expect(configSpec).ToNot(BeNil()) Expect(configSpec.ExtraConfig).To(HaveLen(2)) - extraConfig := session.ExtraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig["guestinfo.foo"]).To(Equal("bar")) Expect(extraConfig["guestinfo.hello"]).To(Equal("world")) }) @@ -313,7 +313,7 @@ var _ = Describe("Cloud-Init Customization", func() { It("ConfigSpec.ExtraConfig to only have metadata", func() { Expect(configSpec).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) - extraConfig := session.ExtraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(2)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -327,7 +327,7 @@ var _ = Describe("Cloud-Init Customization", func() { It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { Expect(configSpec).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) - extraConfig := session.ExtraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -343,7 +343,7 @@ var _ = Describe("Cloud-Init Customization", func() { It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { Expect(configSpec).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) - extraConfig := session.ExtraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -361,7 +361,7 @@ var _ = Describe("Cloud-Init Customization", func() { It("ConfigSpec.ExtraConfig to have metadata and userdata", func() { Expect(configSpec).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) - extraConfig := session.ExtraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -378,7 +378,7 @@ var _ = Describe("Cloud-Init Customization", func() { It("the 'user-data' key overrides as the ConfigSpec.ExtraConfig userdata ", func() { Expect(configSpec).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) - extraConfig := session.ExtraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -394,7 +394,7 @@ var _ = Describe("Cloud-Init Customization", func() { It("ConfigSpec.ExtraConfig's userdata will have values from the 'value' key", func() { Expect(configSpec).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) - extraConfig := session.ExtraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) diff --git a/pkg/vmprovider/providers/vsphere/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere/session/session_vm_update.go index 8626c6d34..be1cf70da 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere/session/session_vm_update.go @@ -329,17 +329,17 @@ func UpdateConfigSpecExtraConfig( if lib.IsVMClassAsConfigFSSDaynDateEnabled() { // Merge non intersecting keys from the desired config spec extra config with the class config spec extra config // (ie) class config spec extra config keys takes precedence over the desired config spec extra config keys - ecFromClassConfigSpec := ExtraConfigToMap(classConfigSpec.ExtraConfig) + ecFromClassConfigSpec := util.ExtraConfigToMap(classConfigSpec.ExtraConfig) mergedExtraConfig := classConfigSpec.ExtraConfig for k, v := range extraConfig { if _, exists := ecFromClassConfigSpec[k]; !exists { mergedExtraConfig = append(mergedExtraConfig, &vimTypes.OptionValue{Key: k, Value: v}) } } - extraConfig = ExtraConfigToMap(mergedExtraConfig) + extraConfig = util.ExtraConfigToMap(mergedExtraConfig) } - configSpec.ExtraConfig = MergeExtraConfig(config.ExtraConfig, extraConfig) + configSpec.ExtraConfig = util.MergeExtraConfig(config.ExtraConfig, extraConfig) // Enabling the defer-cloud-init extraConfig key for V1Alpha1Compatible images defers cloud-init from running on first boot // and disables networking configurations by cloud-init. Therefore, only set the extraConfig key to enabled @@ -348,7 +348,7 @@ func UpdateConfigSpecExtraConfig( // If a VM is deployed from an incompatible image, // it will do nothing and won't cause any issues, but can introduce confusion. if vm.Spec.VmMetadata == nil || vm.Spec.VmMetadata.Transport != vmopv1.VirtualMachineMetadataCloudInitTransport { - ecMap := ExtraConfigToMap(config.ExtraConfig) + ecMap := util.ExtraConfigToMap(config.ExtraConfig) if ecMap[constants.VMOperatorV1Alpha1ExtraConfigKey] == constants.VMOperatorV1Alpha1ConfigReady && imageV1Alpha1Compatible { // Set VMOperatorV1Alpha1ExtraConfigKey for v1alpha1 VirtualMachineImage compatibility. diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go b/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go index 96b71d747..1ce978917 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go @@ -10,13 +10,13 @@ import ( "sigs.k8s.io/yaml" "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/util" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" - res "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/resources" ) type VMDiskData struct { @@ -35,51 +35,101 @@ func BackupVirtualMachine( vmCtx context.VirtualMachineContext, vcVM *object.VirtualMachine, bootstrapData map[string]string) error { - vmKubeData, err := getEncodedVMKubeData(vmCtx.VM) + var moVM mo.VirtualMachine + if err := vcVM.Properties(vmCtx, vcVM.Reference(), + []string{"config.extraConfig"}, &moVM); err != nil { + vmCtx.Logger.Error(err, "Failed to get VM properties for backup") + return err + } + curEcMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig) + + var ecToUpdate []types.BaseOptionValue + + vmKubeDataBackup, err := getDesiredVMKubeDataForBackup(vmCtx.VM, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get encoded VM kube data") + vmCtx.Logger.Error(err, "Failed to get VM kube data for backup") return err } + if vmKubeDataBackup == "" { + vmCtx.Logger.V(4).Info("Skipping VM kube data backup as unchanged") + } else { + ecToUpdate = append(ecToUpdate, &types.OptionValue{ + Key: constants.BackupVMKubeDataExtraConfigKey, + Value: vmKubeDataBackup, + }) + } - vmBootstrapData, err := getEncodedVMBootstrapData(bootstrapData) + instanceIDBackup, err := getDesiredCloudInitInstanceIDForBackup(vmCtx.VM, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get encoded VM bootstrap data") + vmCtx.Logger.Error(err, "Failed to get cloud-init instance ID for backup") return err } - if len(vmBootstrapData) == 0 { - vmCtx.Logger.Info("No VM bootstrap data is provided for backup") + if instanceIDBackup == "" { + vmCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored") + } else { + ecToUpdate = append(ecToUpdate, &types.OptionValue{ + Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, + Value: instanceIDBackup, + }) } - vmDiskData, err := getEncodedVMDiskData(vmCtx, vcVM) + bootstrapDataBackup, err := getDesiredBootstrapDataForBackup(bootstrapData, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get encoded VM disk data") + vmCtx.Logger.Error(err, "Failed to get VM bootstrap data for backup") return err } + if bootstrapDataBackup == "" { + vmCtx.Logger.V(4).Info("Skipping VM bootstrap data backup as unchanged") + } else { + ecToUpdate = append(ecToUpdate, &types.OptionValue{ + Key: constants.BackupVMBootstrapDataExtraConfigKey, + Value: bootstrapDataBackup, + }) + } - _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ - ExtraConfig: []types.BaseOptionValue{ - &types.OptionValue{ - Key: constants.BackupVMKubeDataExtraConfigKey, - Value: vmKubeData, - }, - &types.OptionValue{ - Key: constants.BackupVMBootstrapDataExtraConfigKey, - Value: vmBootstrapData, - }, - &types.OptionValue{ - Key: constants.BackupVMDiskDataExtraConfigKey, - Value: vmDiskData, - }, - }}) + diskDataBackup, err := getDesiredDiskDataForBackup(vmCtx, vcVM, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to reconfigure VM ExtraConfig for backup") + vmCtx.Logger.Error(err, "Failed to get VM disk data for backup") return err } + if diskDataBackup == "" { + vmCtx.Logger.V(4).Info("Skipping VM disk data backup as unchanged") + } else { + ecToUpdate = append(ecToUpdate, &types.OptionValue{ + Key: constants.BackupVMDiskDataExtraConfigKey, + Value: diskDataBackup, + }) + } + + if len(ecToUpdate) != 0 { + vmCtx.Logger.Info("Updating VM ExtraConfig with backup data") + vmCtx.Logger.V(4).Info("", "ExtraConfig", ecToUpdate) + if _, err := vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ExtraConfig: ecToUpdate, + }); err != nil { + vmCtx.Logger.Error(err, "Failed to update VM ExtraConfig for backup") + return err + } + } return nil } -func getEncodedVMKubeData(vm *vmopv1.VirtualMachine) (string, error) { +func getDesiredVMKubeDataForBackup( + vm *vmopv1.VirtualMachine, + ecMap map[string]string) (string, error) { + // If the ExtraConfig already contains the latest VM spec, determined by + // 'metadata.generation', return an empty string to skip the backup. + if ecKubeData, ok := ecMap[constants.BackupVMKubeDataExtraConfigKey]; ok { + vmFromBackup, err := constructVMObj(ecKubeData) + if err != nil { + return "", err + } + if vmFromBackup.ObjectMeta.Generation >= vm.ObjectMeta.Generation { + return "", nil + } + } + backupVM := vm.DeepCopy() backupVM.Status = vmopv1.VirtualMachineStatus{} backupVMYaml, err := yaml.Marshal(backupVM) @@ -90,29 +140,70 @@ func getEncodedVMKubeData(vm *vmopv1.VirtualMachine) (string, error) { return util.EncodeGzipBase64(string(backupVMYaml)) } -func getEncodedVMBootstrapData(bootstrapData map[string]string) (string, error) { - if len(bootstrapData) == 0 { +func constructVMObj(ecKubeData string) (vmopv1.VirtualMachine, error) { + var vmObj vmopv1.VirtualMachine + decodedKubeData, err := util.TryToDecodeBase64Gzip([]byte(ecKubeData)) + if err != nil { + return vmObj, err + } + + err = yaml.Unmarshal([]byte(decodedKubeData), &vmObj) + return vmObj, err +} + +func getDesiredCloudInitInstanceIDForBackup( + vm *vmopv1.VirtualMachine, + ecMap map[string]string) (string, error) { + // Cloud-Init instance ID should not be changed once persisted in VM's + // ExtraConfig. Return an empty string to skip the backup if it exists. + if _, ok := ecMap[constants.BackupVMCloudInitInstanceIDExtraConfigKey]; ok { return "", nil } - bootstrapDataJSON, err := json.Marshal(bootstrapData) + instanceID := vm.Annotations[vmopv1.InstanceIDAnnotation] + if instanceID == "" { + instanceID = string(vm.UID) + } + + return util.EncodeGzipBase64(instanceID) +} + +func getDesiredBootstrapDataForBackup( + bootstrapDataRaw map[string]string, + ecMap map[string]string) (string, error) { + // No bootstrap data is specified, return an empty string to skip the backup. + if len(bootstrapDataRaw) == 0 { + return "", nil + } + + bootstrapDataJSON, err := json.Marshal(bootstrapDataRaw) + if err != nil { + return "", err + } + bootstrapDataBackup, err := util.EncodeGzipBase64(string(bootstrapDataJSON)) if err != nil { return "", err } - return util.EncodeGzipBase64(string(bootstrapDataJSON)) + // Return an empty string to skip the backup if the data is unchanged. + if bootstrapDataBackup == ecMap[constants.BackupVMBootstrapDataExtraConfigKey] { + return "", nil + } + + return bootstrapDataBackup, nil } -func getEncodedVMDiskData( - ctx goctx.Context, vcVM *object.VirtualMachine) (string, error) { - resVM := res.NewVMFromObject(vcVM) - disks, err := resVM.GetVirtualDisks(ctx) +func getDesiredDiskDataForBackup( + ctx goctx.Context, + vcVM *object.VirtualMachine, + ecMap map[string]string) (string, error) { + deviceList, err := vcVM.Device(ctx) if err != nil { return "", err } - diskData := []VMDiskData{} - for _, device := range disks { + var diskData []VMDiskData + for _, device := range deviceList.SelectByType((*types.VirtualDisk)(nil)) { if disk, ok := device.(*types.VirtualDisk); ok { vmDiskData := VMDiskData{} if disk.VDiskId != nil { @@ -132,6 +223,15 @@ func getEncodedVMDiskData( if err != nil { return "", err } + diskDataBackup, err := util.EncodeGzipBase64(string(diskDataJSON)) + if err != nil { + return "", err + } + + // Return an empty string to skip the backup if the data is unchanged. + if diskDataBackup == ecMap[constants.BackupVMDiskDataExtraConfigKey] { + return "", nil + } - return util.EncodeGzipBase64(string(diskDataJSON)) + return diskDataBackup, nil } diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go b/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go index dfa40ca18..13a0aad9c 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go @@ -12,12 +12,12 @@ import ( . "github.com/onsi/gomega" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/util" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" - "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/session" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/virtualmachine" "github.com/vmware-tanzu/vm-operator/test/builder" ) @@ -49,29 +49,82 @@ func backupTests() { }) Context("VM Kube data", func() { - BeforeEach(func() { vmCtx.VM = builder.DummyVirtualMachine() }) - It("Should backup VM kube data YAML without status field in ExtraConfig", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + When("VM kube data exists in ExtraConfig but is not up-to-date", func() { + + BeforeEach(func() { + oldVM := vmCtx.VM.DeepCopy() + oldVM.ObjectMeta.Generation = 1 + oldVMYaml, err := yaml.Marshal(oldVM) + Expect(err).NotTo(HaveOccurred()) + backupVMYamlEncoded, err := util.EncodeGzipBase64(string(oldVMYaml)) + Expect(err).NotTo(HaveOccurred()) + + _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{ + Key: constants.BackupVMKubeDataExtraConfigKey, + Value: backupVMYamlEncoded, + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Should backup VM kube data YAML with the latest spec", func() { + vmCtx.VM.ObjectMeta.Generation = 2 + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + + vmCopy := vmCtx.VM.DeepCopy() + vmCopy.Status = vmopv1.VirtualMachineStatus{} + vmCopyYaml, err := yaml.Marshal(vmCopy) + Expect(err).NotTo(HaveOccurred()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, string(vmCopyYaml)) + }) + }) - vmCopy := vmCtx.VM.DeepCopy() - vmCopy.Status = vmopv1.VirtualMachineStatus{} - vmCopyYaml, err := yaml.Marshal(vmCopy) - Expect(err).NotTo(HaveOccurred()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, string(vmCopyYaml)) + When("VM kube data exists in ExtraConfig and is up-to-date", func() { + var ( + kubeDataBackup = "" + ) + + BeforeEach(func() { + vmYaml, err := yaml.Marshal(vmCtx.VM) + Expect(err).NotTo(HaveOccurred()) + kubeDataBackup = string(vmYaml) + encodedKubeDataBackup, err := util.EncodeGzipBase64(kubeDataBackup) + Expect(err).NotTo(HaveOccurred()) + + _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{ + Key: constants.BackupVMKubeDataExtraConfigKey, + Value: encodedKubeDataBackup, + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Should skip backing up VM kube data", func() { + // Update the VM to verify its kube data is not backed up in ExtraConfig. + vmCtx.VM.Labels = map[string]string{"foo": "bar"} + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, kubeDataBackup) + }) }) }) Context("VM bootstrap data", func() { It("Should back up bootstrap data as JSON in ExtraConfig", func() { - bootstrapData := map[string]string{"foo": "bar"} - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, bootstrapData)).To(Succeed()) + bootstrapDataRaw := map[string]string{"foo": "bar"} + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, bootstrapDataRaw)).To(Succeed()) - bootstrapDataJSON, err := json.Marshal(bootstrapData) + bootstrapDataJSON, err := json.Marshal(bootstrapDataRaw) Expect(err).NotTo(HaveOccurred()) verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMBootstrapDataExtraConfigKey, string(bootstrapDataJSON)) }) @@ -94,6 +147,64 @@ func backupTests() { verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, string(diskDataJSON)) }) }) + + Context("VM cloud-init instance ID data", func() { + + BeforeEach(func() { + vmCtx.VM = builder.DummyVirtualMachine() + }) + + When("VM cloud-init instance ID already exists in ExtraConfig", func() { + + BeforeEach(func() { + _, err := vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{ + Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, + Value: "ec-instance-id", + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + vmCtx.VM.Annotations = map[string]string{ + vmopv1.InstanceIDAnnotation: "other-instance-id", + } + }) + + It("Should skip backing up the cloud-init instance ID", func() { + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "ec-instance-id") + }) + }) + + When("VM cloud-init instance ID does not exist in ExtraConfig and is set in annotations", func() { + + BeforeEach(func() { + vmCtx.VM.Annotations = map[string]string{ + vmopv1.InstanceIDAnnotation: "annotation-instance-id", + } + vmCtx.VM.UID = "vm-uid" + }) + + It("Should backup the cloud-init instance ID from annotations", func() { + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "annotation-instance-id") + }) + }) + + When("VM cloud-init instance ID does not exist in ExtraConfig and is not set in annotations", func() { + + BeforeEach(func() { + vmCtx.VM.Annotations = nil + vmCtx.VM.UID = "vm-uid" + }) + + It("Should backup the cloud-init instance ID from annotations", func() { + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "vm-uid") + }) + }) + }) } func verifyBackupDataInExtraConfig( @@ -107,7 +218,7 @@ func verifyBackupDataInExtraConfig( Expect(objVM).NotTo(BeNil()) var moVM mo.VirtualMachine Expect(objVM.Properties(ctx, objVM.Reference(), []string{"config.extraConfig"}, &moVM)).To(Succeed()) - ecMap := session.ExtraConfigToMap(moVM.Config.ExtraConfig) + ecMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig) // Verify the expected key exists in ExtraConfig and the decoded values match. Expect(ecMap).To(HaveKey(expectedKey)) diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm.go index 15ce347da..4a012cdf6 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm.go @@ -131,32 +131,6 @@ func (vs *vSphereVMProvider) PublishVirtualMachine(ctx goctx.Context, vm *vmopv1 return itemID, nil } -// BackupVirtualMachine backs up the VM data required for restore. -func (vs *vSphereVMProvider) BackupVirtualMachine(ctx goctx.Context, vm *vmopv1.VirtualMachine) error { - client, err := vs.getVcClient(ctx) - if err != nil { - return errors.Wrapf(err, "failed to get vCenter client for backing up Virtual Machine") - } - - vmCtx := context.VirtualMachineContext{ - Context: goctx.WithValue(ctx, types.ID{}, vs.getOpID(vm, "backupVM")), - Logger: log.WithValues("vmName", vm.NamespacedName()), - VM: vm, - } - - vcVM, err := vs.getVM(vmCtx, client, true) - if err != nil { - return errors.Wrapf(err, "failed to get vSphere Virtual Machine for backing up") - } - - vmMetadata, err := GetVMMetadata(vmCtx, vs.k8sClient) - if err != nil { - return errors.Wrapf(err, "failed to get VM metadata for backing up") - } - - return virtualmachine.BackupVirtualMachine(vmCtx, vcVM, vmMetadata.Data) -} - func (vs *vSphereVMProvider) GetVirtualMachineGuestHeartbeat( ctx goctx.Context, vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { @@ -342,6 +316,18 @@ func (vs *vSphereVMProvider) updateVirtualMachine( } } + // Back up the VM at the end after a successful update. + if lib.IsVMServiceBackupRestoreFSSEnabled() { + vmCtx.Logger.V(4).Info("Backing up VirtualMachine") + data, err := GetVMMetadata(vmCtx, vs.k8sClient) + if err != nil { + return err + } + if err := virtualmachine.BackupVirtualMachine(vmCtx, vcVM, data.Data); err != nil { + return err + } + } + return nil } diff --git a/pkg/vmprovider/providers/vsphere2/session/session_util.go b/pkg/vmprovider/providers/vsphere2/session/session_util.go deleted file mode 100644 index fb25dbe91..000000000 --- a/pkg/vmprovider/providers/vsphere2/session/session_util.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2018-2022 VMware, Inc. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package session - -import ( - vimTypes "github.com/vmware/govmomi/vim25/types" -) - -func ExtraConfigToMap(input []vimTypes.BaseOptionValue) (output map[string]string) { - output = make(map[string]string) - for _, opt := range input { - if optValue := opt.GetOptionValue(); optValue != nil { - // Only set string type values - if val, ok := optValue.Value.(string); ok { - output[optValue.Key] = val - } - } - } - return -} - -// MergeExtraConfig adds the key/value to the ExtraConfig if the key is not present, to let to the value be -// changed by the VM. The existing usage of ExtraConfig is hard to fit in the reconciliation model. -func MergeExtraConfig(extraConfig []vimTypes.BaseOptionValue, newMap map[string]string) []vimTypes.BaseOptionValue { - merged := make([]vimTypes.BaseOptionValue, 0) - ecMap := ExtraConfigToMap(extraConfig) - for k, v := range newMap { - if _, exists := ecMap[k]; !exists { - merged = append(merged, &vimTypes.OptionValue{Key: k, Value: v}) - } - } - return merged -} diff --git a/pkg/vmprovider/providers/vsphere2/session/session_util_test.go b/pkg/vmprovider/providers/vsphere2/session/session_util_test.go deleted file mode 100644 index 76ca2a0fb..000000000 --- a/pkg/vmprovider/providers/vsphere2/session/session_util_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) 2019-2022 VMware, Inc. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package session_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - vimTypes "github.com/vmware/govmomi/vim25/types" - - "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/session" -) - -var _ = Describe("Test Session Utils", func() { - - Context("ExtraConfigToMap", func() { - var ( - extraConfig []vimTypes.BaseOptionValue - extraConfigMap map[string]string - ) - BeforeEach(func() { - extraConfig = []vimTypes.BaseOptionValue{} - }) - JustBeforeEach(func() { - extraConfigMap = session.ExtraConfigToMap(extraConfig) - }) - - Context("Empty extraConfig", func() { - It("Return empty map", func() { - Expect(extraConfigMap).To(HaveLen(0)) - }) - }) - - Context("With extraConfig", func() { - BeforeEach(func() { - extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key1", Value: "value1"}) - extraConfig = append(extraConfig, &vimTypes.OptionValue{Key: "key2", Value: "value2"}) - }) - It("Return valid map", func() { - Expect(extraConfigMap).To(HaveLen(2)) - Expect(extraConfigMap["key1"]).To(Equal("value1")) - Expect(extraConfigMap["key2"]).To(Equal("value2")) - }) - }) - }) - - Context("MergeExtraConfig", func() { - var ( - extraConfig []vimTypes.BaseOptionValue - newMap map[string]string - merged []vimTypes.BaseOptionValue - ) - BeforeEach(func() { - extraConfig = []vimTypes.BaseOptionValue{ - &vimTypes.OptionValue{Key: "existingkey1", Value: "existingvalue1"}, - &vimTypes.OptionValue{Key: "existingkey2", Value: "existingvalue2"}, - } - newMap = map[string]string{} - }) - JustBeforeEach(func() { - merged = session.MergeExtraConfig(extraConfig, newMap) - }) - - Context("Empty newMap", func() { - It("Return empty merged", func() { - Expect(merged).To(BeEmpty()) - }) - }) - - Context("NewMap with existing key", func() { - BeforeEach(func() { - newMap["existingkey1"] = "existingkey1" - }) - It("Return empty merged", func() { - Expect(merged).To(BeEmpty()) - }) - }) - - Context("NewMap with new keys", func() { - BeforeEach(func() { - newMap["newkey1"] = "newvalue1" - newMap["newkey2"] = "newvalue2" - }) - It("Return merged map", func() { - Expect(merged).To(HaveLen(2)) - mergedMap := session.ExtraConfigToMap(merged) - Expect(mergedMap["newkey1"]).To(Equal("newvalue1")) - Expect(mergedMap["newkey2"]).To(Equal("newvalue2")) - }) - }) - }) -}) diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go index 81cef4856..baf4e5c74 100644 --- a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go @@ -320,17 +320,17 @@ func UpdateConfigSpecExtraConfig( if lib.IsVMClassAsConfigFSSDaynDateEnabled() { // Merge non intersecting keys from the desired config spec extra config with the class config spec extra config // (ie) class config spec extra config keys takes precedence over the desired config spec extra config keys - ecFromClassConfigSpec := ExtraConfigToMap(classConfigSpec.ExtraConfig) + ecFromClassConfigSpec := util.ExtraConfigToMap(classConfigSpec.ExtraConfig) mergedExtraConfig := classConfigSpec.ExtraConfig for k, v := range extraConfig { if _, exists := ecFromClassConfigSpec[k]; !exists { mergedExtraConfig = append(mergedExtraConfig, &vimTypes.OptionValue{Key: k, Value: v}) } } - extraConfig = ExtraConfigToMap(mergedExtraConfig) + extraConfig = util.ExtraConfigToMap(mergedExtraConfig) } - configSpec.ExtraConfig = MergeExtraConfig(config.ExtraConfig, extraConfig) + configSpec.ExtraConfig = util.MergeExtraConfig(config.ExtraConfig, extraConfig) // Enabling the defer-cloud-init extraConfig key for V1Alpha1Compatible images defers cloud-init from running on first boot // and disables networking configurations by cloud-init. Therefore, only set the extraConfig key to enabled @@ -341,7 +341,7 @@ func UpdateConfigSpecExtraConfig( // BMV: Is this needed anymore? IMO we shouldn't have bootstrap stuff here. The EC mangling is already hard to follow. emptyBSSpec := vmopv1.VirtualMachineBootstrapSpec{} if vm.Spec.Bootstrap == emptyBSSpec || vm.Spec.Bootstrap.CloudInit == nil { - ecMap := ExtraConfigToMap(config.ExtraConfig) + ecMap := util.ExtraConfigToMap(config.ExtraConfig) if ecMap[constants.VMOperatorV1Alpha1ExtraConfigKey] == constants.VMOperatorV1Alpha1ConfigReady && imageV1Alpha1Compatible { // Set VMOperatorV1Alpha1ExtraConfigKey for v1alpha1 VirtualMachineImage compatibility. diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go index 7badf04c8..82d376e22 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_cloudinit_test.go @@ -50,19 +50,6 @@ var _ = Describe("CloudInit Bootstrap", func() { bsArgs = vmlifecycle.BootstrapArgs{} }) - extraConfigToMap := func(input []types.BaseOptionValue) map[string]string { - output := make(map[string]string) - for _, opt := range input { - if optValue := opt.GetOptionValue(); optValue != nil { - // Only set string type values - if val, ok := optValue.Value.(string); ok { - output[optValue.Key] = val - } - } - } - return output - } - // v1a1 tests really only tested the lower level functions individually. Those tests are ported after // this Context, but we should focus more on testing via this just method. Context("BootStrapCloudInit", func() { @@ -151,7 +138,7 @@ var _ = Describe("CloudInit Bootstrap", func() { Expect(configSpec.VAppConfigRemoved).ToNot(BeNil()) Expect(*configSpec.VAppConfigRemoved).To(BeTrue()) - extraConfig := extraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoMetadata)) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -174,7 +161,7 @@ var _ = Describe("CloudInit Bootstrap", func() { Expect(err).ToNot(HaveOccurred()) Expect(configSpec).ToNot(BeNil()) - extraConfig := extraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig).To(HaveKey(constants.CloudInitGuestInfoMetadata)) // TODO: Better assertion (reduce w/ GetCloudInitMetadata) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -264,7 +251,7 @@ var _ = Describe("CloudInit Bootstrap", func() { Expect(configSpec).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) - extraConfig := extraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(2)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -276,7 +263,7 @@ var _ = Describe("CloudInit Bootstrap", func() { Expect(err).ToNot(HaveOccurred()) Expect(configSpec).ToNot(BeNil()) - extraConfig := extraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -294,7 +281,7 @@ var _ = Describe("CloudInit Bootstrap", func() { Expect(err).ToNot(HaveOccurred()) Expect(configSpec).ToNot(BeNil()) - extraConfig := extraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) @@ -314,7 +301,7 @@ var _ = Describe("CloudInit Bootstrap", func() { Expect(err).ToNot(HaveOccurred()) Expect(configSpec).ToNot(BeNil()) - extraConfig := extraConfigToMap(configSpec.ExtraConfig) + extraConfig := util.ExtraConfigToMap(configSpec.ExtraConfig) Expect(extraConfig).To(HaveLen(4)) Expect(extraConfig[constants.CloudInitGuestInfoMetadata]).To(Equal("H4sIAAAAAAAA/0rOyS9N0c3MyyzRzU0tSUxJLEkEAAAA//8BAAD//wEq0o4TAAAA")) Expect(extraConfig[constants.CloudInitGuestInfoMetadataEncoding]).To(Equal("gzip+base64")) From b5dda1ba7b9f5d80401226ce0e4f372374661f5b Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Wed, 18 Oct 2023 17:42:48 -0400 Subject: [PATCH 33/54] =?UTF-8?q?=F0=9F=8C=B1=20Add=20VM=20backup=20implem?= =?UTF-8?q?entation=20for=20v1a2=20(#240)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds the implementation for VM backup in v1a2. It includes all the follow-up changes in PR #239 with some minor updates specific to v1a2. --- .../v1alpha2/virtualmachine_controller.go | 9 - .../virtualmachine_controller_unit_test.go | 23 -- pkg/vmprovider/fake/fake_vm_provider_a2.go | 12 - pkg/vmprovider/interface_a2.go | 1 - .../providers/vsphere2/constants/constants.go | 13 + .../vsphere2/virtualmachine/backup.go | 237 ++++++++++++++++++ .../vsphere2/virtualmachine/backup_test.go | 229 +++++++++++++++++ .../virtualmachine_suite_test.go | 1 + .../providers/vsphere2/vmprovider_vm.go | 19 +- 9 files changed, 493 insertions(+), 51 deletions(-) create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go create mode 100644 pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go diff --git a/controllers/virtualmachine/v1alpha2/virtualmachine_controller.go b/controllers/virtualmachine/v1alpha2/virtualmachine_controller.go index 838d3a0ac..ceaea80f9 100644 --- a/controllers/virtualmachine/v1alpha2/virtualmachine_controller.go +++ b/controllers/virtualmachine/v1alpha2/virtualmachine_controller.go @@ -336,15 +336,6 @@ func (r *Reconciler) ReconcileNormal(ctx *context.VirtualMachineContextA2) (rete // Add this VM to prober manager if ReconcileNormal succeeds. r.Prober.AddToProberManager(ctx.VM) - // Back up this VM if ReconcileNormal succeeds and the FSS is enabled. - if lib.IsVMServiceBackupRestoreFSSEnabled() { - if err := r.VMProvider.BackupVirtualMachine(ctx, ctx.VM); err != nil { - ctx.Logger.Error(err, "Failed to backup VirtualMachine") - r.Recorder.EmitEvent(ctx.VM, "Backup", err, false) - return err - } - } - ctx.Logger.Info("Finished Reconciling VirtualMachine") return nil } diff --git a/controllers/virtualmachine/v1alpha2/virtualmachine_controller_unit_test.go b/controllers/virtualmachine/v1alpha2/virtualmachine_controller_unit_test.go index f9f8a6f07..80d7d1a68 100644 --- a/controllers/virtualmachine/v1alpha2/virtualmachine_controller_unit_test.go +++ b/controllers/virtualmachine/v1alpha2/virtualmachine_controller_unit_test.go @@ -6,7 +6,6 @@ package v1alpha2_test import ( "context" "errors" - "os" "strings" . "github.com/onsi/ginkgo" @@ -19,7 +18,6 @@ import ( virtualmachine "github.com/vmware-tanzu/vm-operator/controllers/virtualmachine/v1alpha2" vmopContext "github.com/vmware-tanzu/vm-operator/pkg/context" - "github.com/vmware-tanzu/vm-operator/pkg/lib" proberfake "github.com/vmware-tanzu/vm-operator/pkg/prober2/fake" providerfake "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/fake" "github.com/vmware-tanzu/vm-operator/test/builder" @@ -155,27 +153,6 @@ func unitTestsReconcile() { Expect(reconciler.ReconcileNormal(vmCtx)).Should(Succeed()) Expect(fakeProbeManager.IsAddToProberManagerCalled).Should(BeTrue()) }) - - When("The VM Service Backup and Restore FSS is enabled", func() { - BeforeEach(func() { - Expect(os.Setenv(lib.VMServiceBackupRestoreFSS, lib.TrueString)).To(Succeed()) - }) - - AfterEach(func() { - Expect(os.Unsetenv(lib.VMServiceBackupRestoreFSS)).To(Succeed()) - }) - - It("Should call backup Virtual Machine if ReconcileNormal succeeds", func() { - var isBackupVirtualMachineCalled bool - fakeVMProvider.BackupVirtualMachineFn = func(ctx context.Context, vm *vmopv1.VirtualMachine) error { - isBackupVirtualMachineCalled = true - return nil - } - - Expect(reconciler.ReconcileNormal(vmCtx)).Should(Succeed()) - Expect(isBackupVirtualMachineCalled).Should(BeTrue()) - }) - }) }) Context("ReconcileDelete", func() { diff --git a/pkg/vmprovider/fake/fake_vm_provider_a2.go b/pkg/vmprovider/fake/fake_vm_provider_a2.go index ccbea29fe..b0e8dcf1f 100644 --- a/pkg/vmprovider/fake/fake_vm_provider_a2.go +++ b/pkg/vmprovider/fake/fake_vm_provider_a2.go @@ -31,7 +31,6 @@ type funcsA2 struct { DeleteVirtualMachineFn func(ctx context.Context, vm *vmopv1.VirtualMachine) error PublishVirtualMachineFn func(ctx context.Context, vm *vmopv1.VirtualMachine, vmPub *vmopv1.VirtualMachinePublishRequest, cl *imgregv1a1.ContentLibrary, actID string) (string, error) - BackupVirtualMachineFn func(ctx context.Context, vm *vmopv1.VirtualMachine) error GetVirtualMachineGuestHeartbeatFn func(ctx context.Context, vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) GetVirtualMachineWebMKSTicketFn func(ctx context.Context, vm *vmopv1.VirtualMachine, pubKey string) (string, error) GetVirtualMachineHardwareVersionFn func(ctx context.Context, vm *vmopv1.VirtualMachine) (int32, error) @@ -113,17 +112,6 @@ func (s *VMProviderA2) PublishVirtualMachine(ctx context.Context, vm *vmopv1.Vir return "dummy-id", nil } -func (s *VMProviderA2) BackupVirtualMachine(ctx context.Context, vm *vmopv1.VirtualMachine) error { - s.Lock() - defer s.Unlock() - - if s.BackupVirtualMachineFn != nil { - return s.BackupVirtualMachineFn(ctx, vm) - } - - return nil -} - func (s *VMProviderA2) GetVirtualMachineGuestHeartbeat(ctx context.Context, vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { s.Lock() defer s.Unlock() diff --git a/pkg/vmprovider/interface_a2.go b/pkg/vmprovider/interface_a2.go index 784d47afc..2b9819c26 100644 --- a/pkg/vmprovider/interface_a2.go +++ b/pkg/vmprovider/interface_a2.go @@ -21,7 +21,6 @@ type VirtualMachineProviderInterfaceA2 interface { DeleteVirtualMachine(ctx context.Context, vm *v1alpha2.VirtualMachine) error PublishVirtualMachine(ctx context.Context, vm *v1alpha2.VirtualMachine, vmPub *v1alpha2.VirtualMachinePublishRequest, cl *imgregv1a1.ContentLibrary, actID string) (string, error) - BackupVirtualMachine(ctx context.Context, vm *v1alpha2.VirtualMachine) error GetVirtualMachineGuestHeartbeat(ctx context.Context, vm *v1alpha2.VirtualMachine) (v1alpha2.GuestHeartbeatStatus, error) GetVirtualMachineWebMKSTicket(ctx context.Context, vm *v1alpha2.VirtualMachine, pubKey string) (string, error) GetVirtualMachineHardwareVersion(ctx context.Context, vm *v1alpha2.VirtualMachine) (int32, error) diff --git a/pkg/vmprovider/providers/vsphere2/constants/constants.go b/pkg/vmprovider/providers/vsphere2/constants/constants.go index ef26052b7..725b1ad0c 100644 --- a/pkg/vmprovider/providers/vsphere2/constants/constants.go +++ b/pkg/vmprovider/providers/vsphere2/constants/constants.go @@ -134,4 +134,17 @@ const ( V1alpha2SubnetMask = "V1alpha2_SubnetMask" // V1alpha2FormatNameservers is an alias for versioned templating function V1alpha2_FormatNameservers. V1alpha2FormatNameservers = "V1alpha2_FormatNameservers" + + // BackupVMKubeDataExtraConfigKey is the ExtraConfig key to the VirtualMachine + // resource's Kubernetes spec data, compressed using gzip and base64-encoded. + BackupVMKubeDataExtraConfigKey = "vmservice.virtualmachine.kubedata" + // BackupVMBootstrapDataExtraConfigKey is the ExtraConfig key to the VM's + // bootstrap data object, compressed using gzip and base64-encoded. + BackupVMBootstrapDataExtraConfigKey = "vmservice.virtualmachine.bootstrapdata" + // BackupVMDiskDataExtraConfigKey is the ExtraConfig key to the VM's disk info + // data in JSON, compressed using gzip and base64-encoded. + BackupVMDiskDataExtraConfigKey = "vmservice.virtualmachine.diskdata" + // BackupVMCloudInitInstanceIDExtraConfigKey is the ExtraConfig key to the VM's + // Cloud-Init instance ID, compressed using gzip and base64-encoded. + BackupVMCloudInitInstanceIDExtraConfigKey = "vmservice.virtualmachine.cloudinit.instanceid" ) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go new file mode 100644 index 000000000..bf40f2181 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go @@ -0,0 +1,237 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine + +import ( + goctx "context" + "encoding/json" + + "sigs.k8s.io/yaml" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" +) + +type VMDiskData struct { + // ID of the virtual disk object (only set for FCDs). + VDiskID string + // Filename contains the datastore path to the virtual disk. + FileName string +} + +// BackupVirtualMachine backs up the required data of a VM into its ExtraConfig. +// Currently, the following data is backed up: +// - Kubernetes VirtualMachine object in YAML format (without its .status field). +// - VM bootstrap data in JSON (if provided). +// - List of VM disk data in JSON (including FCDs attached to the VM). +func BackupVirtualMachine( + vmCtx context.VirtualMachineContextA2, + vcVM *object.VirtualMachine, + bootstrapData map[string]string) error { + var moVM mo.VirtualMachine + if err := vcVM.Properties(vmCtx, vcVM.Reference(), + []string{"config.extraConfig"}, &moVM); err != nil { + vmCtx.Logger.Error(err, "Failed to get VM properties for backup") + return err + } + curEcMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig) + + var ecToUpdate []types.BaseOptionValue + + vmKubeDataBackup, err := getDesiredVMKubeDataForBackup(vmCtx.VM, curEcMap) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get VM kube data for backup") + return err + } + if vmKubeDataBackup == "" { + vmCtx.Logger.V(4).Info("Skipping VM kube data backup as unchanged") + } else { + ecToUpdate = append(ecToUpdate, &types.OptionValue{ + Key: constants.BackupVMKubeDataExtraConfigKey, + Value: vmKubeDataBackup, + }) + } + + instanceIDBackup, err := getDesiredCloudInitInstanceIDForBackup(vmCtx.VM, curEcMap) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get cloud-init instance ID for backup") + return err + } + if instanceIDBackup == "" { + vmCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored") + } else { + ecToUpdate = append(ecToUpdate, &types.OptionValue{ + Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, + Value: instanceIDBackup, + }) + } + + bootstrapDataBackup, err := getDesiredBootstrapDataForBackup(bootstrapData, curEcMap) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get VM bootstrap data for backup") + return err + } + if bootstrapDataBackup == "" { + vmCtx.Logger.V(4).Info("Skipping VM bootstrap data backup as unchanged") + } else { + ecToUpdate = append(ecToUpdate, &types.OptionValue{ + Key: constants.BackupVMBootstrapDataExtraConfigKey, + Value: bootstrapDataBackup, + }) + } + + diskDataBackup, err := getDesiredDiskDataForBackup(vmCtx, vcVM, curEcMap) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get VM disk data for backup") + return err + } + if diskDataBackup == "" { + vmCtx.Logger.V(4).Info("Skipping VM disk data backup as unchanged") + } else { + ecToUpdate = append(ecToUpdate, &types.OptionValue{ + Key: constants.BackupVMDiskDataExtraConfigKey, + Value: diskDataBackup, + }) + } + + if len(ecToUpdate) != 0 { + vmCtx.Logger.Info("Updating VM ExtraConfig with backup data") + vmCtx.Logger.V(4).Info("", "ExtraConfig", ecToUpdate) + if _, err := vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ExtraConfig: ecToUpdate, + }); err != nil { + vmCtx.Logger.Error(err, "Failed to update VM ExtraConfig for backup") + return err + } + } + + return nil +} + +func getDesiredVMKubeDataForBackup( + vm *vmopv1.VirtualMachine, + ecMap map[string]string) (string, error) { + // If the ExtraConfig already contains the latest VM spec, determined by + // 'metadata.generation', return an empty string to skip the backup. + if ecKubeData, ok := ecMap[constants.BackupVMKubeDataExtraConfigKey]; ok { + vmFromBackup, err := constructVMObj(ecKubeData) + if err != nil { + return "", err + } + if vmFromBackup.ObjectMeta.Generation >= vm.ObjectMeta.Generation { + return "", nil + } + } + + backupVM := vm.DeepCopy() + backupVM.Status = vmopv1.VirtualMachineStatus{} + backupVMYaml, err := yaml.Marshal(backupVM) + if err != nil { + return "", err + } + + return util.EncodeGzipBase64(string(backupVMYaml)) +} + +func constructVMObj(ecKubeData string) (vmopv1.VirtualMachine, error) { + var vmObj vmopv1.VirtualMachine + decodedKubeData, err := util.TryToDecodeBase64Gzip([]byte(ecKubeData)) + if err != nil { + return vmObj, err + } + + err = yaml.Unmarshal([]byte(decodedKubeData), &vmObj) + return vmObj, err +} + +func getDesiredCloudInitInstanceIDForBackup( + vm *vmopv1.VirtualMachine, + ecMap map[string]string) (string, error) { + // Cloud-Init instance ID should not be changed once persisted in VM's + // ExtraConfig. Return an empty string to skip the backup if it exists. + if _, ok := ecMap[constants.BackupVMCloudInitInstanceIDExtraConfigKey]; ok { + return "", nil + } + + instanceID := vm.Annotations[vmopv1.InstanceIDAnnotation] + if instanceID == "" { + instanceID = string(vm.UID) + } + + return util.EncodeGzipBase64(instanceID) +} + +func getDesiredBootstrapDataForBackup( + bootstrapDataRaw map[string]string, + ecMap map[string]string) (string, error) { + // No bootstrap data is specified, return an empty string to skip the backup. + if len(bootstrapDataRaw) == 0 { + return "", nil + } + + bootstrapDataJSON, err := json.Marshal(bootstrapDataRaw) + if err != nil { + return "", err + } + bootstrapDataBackup, err := util.EncodeGzipBase64(string(bootstrapDataJSON)) + if err != nil { + return "", err + } + + // Return an empty string to skip the backup if the data is unchanged. + if bootstrapDataBackup == ecMap[constants.BackupVMBootstrapDataExtraConfigKey] { + return "", nil + } + + return bootstrapDataBackup, nil +} + +func getDesiredDiskDataForBackup( + ctx goctx.Context, + vcVM *object.VirtualMachine, + ecMap map[string]string) (string, error) { + deviceList, err := vcVM.Device(ctx) + if err != nil { + return "", err + } + + var diskData []VMDiskData + for _, device := range deviceList.SelectByType((*types.VirtualDisk)(nil)) { + if disk, ok := device.(*types.VirtualDisk); ok { + vmDiskData := VMDiskData{} + if disk.VDiskId != nil { + vmDiskData.VDiskID = disk.VDiskId.Id + } + if b, ok := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok { + vmDiskData.FileName = b.FileName + } + // Only add the disk data if it's not empty. + if vmDiskData != (VMDiskData{}) { + diskData = append(diskData, vmDiskData) + } + } + } + + diskDataJSON, err := json.Marshal(diskData) + if err != nil { + return "", err + } + diskDataBackup, err := util.EncodeGzipBase64(string(diskDataJSON)) + if err != nil { + return "", err + } + + // Return an empty string to skip the backup if the data is unchanged. + if diskDataBackup == ecMap[constants.BackupVMDiskDataExtraConfigKey] { + return "", nil + } + + return diskDataBackup, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go new file mode 100644 index 000000000..d26190ae3 --- /dev/null +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go @@ -0,0 +1,229 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package virtualmachine_test + +import ( + "encoding/json" + + "sigs.k8s.io/yaml" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/util" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" + "github.com/vmware-tanzu/vm-operator/test/builder" +) + +func backupTests() { + var ( + ctx *builder.TestContextForVCSim + vcVM *object.VirtualMachine + vmCtx context.VirtualMachineContextA2 + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{}) + + var err error + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).NotTo(HaveOccurred()) + + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vcVM.Name()), + VM: &vmopv1.VirtualMachine{}, + } + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + Context("VM Kube data", func() { + BeforeEach(func() { + vmCtx.VM = builder.DummyVirtualMachineA2() + }) + + When("VM kube data exists in ExtraConfig but is not up-to-date", func() { + + BeforeEach(func() { + oldVM := vmCtx.VM.DeepCopy() + oldVM.ObjectMeta.Generation = 1 + oldVMYaml, err := yaml.Marshal(oldVM) + Expect(err).NotTo(HaveOccurred()) + backupVMYamlEncoded, err := util.EncodeGzipBase64(string(oldVMYaml)) + Expect(err).NotTo(HaveOccurred()) + + _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{ + Key: constants.BackupVMKubeDataExtraConfigKey, + Value: backupVMYamlEncoded, + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Should backup VM kube data YAML with the latest spec", func() { + vmCtx.VM.ObjectMeta.Generation = 2 + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + + vmCopy := vmCtx.VM.DeepCopy() + vmCopy.Status = vmopv1.VirtualMachineStatus{} + vmCopyYaml, err := yaml.Marshal(vmCopy) + Expect(err).NotTo(HaveOccurred()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, string(vmCopyYaml)) + }) + }) + + When("VM kube data exists in ExtraConfig and is up-to-date", func() { + var ( + kubeDataBackup = "" + ) + + BeforeEach(func() { + vmYaml, err := yaml.Marshal(vmCtx.VM) + Expect(err).NotTo(HaveOccurred()) + kubeDataBackup = string(vmYaml) + encodedKubeDataBackup, err := util.EncodeGzipBase64(kubeDataBackup) + Expect(err).NotTo(HaveOccurred()) + + _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{ + Key: constants.BackupVMKubeDataExtraConfigKey, + Value: encodedKubeDataBackup, + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Should skip backing up VM kube data", func() { + // Update the VM to verify its kube data is not backed up in ExtraConfig. + vmCtx.VM.Labels = map[string]string{"foo": "bar"} + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, kubeDataBackup) + }) + }) + }) + + Context("VM bootstrap data", func() { + + It("Should back up bootstrap data as JSON in ExtraConfig", func() { + bootstrapDataRaw := map[string]string{"foo": "bar"} + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, bootstrapDataRaw)).To(Succeed()) + + bootstrapDataJSON, err := json.Marshal(bootstrapDataRaw) + Expect(err).NotTo(HaveOccurred()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMBootstrapDataExtraConfigKey, string(bootstrapDataJSON)) + }) + }) + + Context("VM Disk data", func() { + + It("Should backup VM disk data as JSON in ExtraConfig", func() { + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + + // Use the default disk info from the vcSim VM for testing. + diskData := []virtualmachine.VMDiskData{ + { + VDiskID: "", + FileName: "[LocalDS_0] DC0_C0_RP0_VM0/disk1.vmdk", + }, + } + diskDataJSON, err := json.Marshal(diskData) + Expect(err).NotTo(HaveOccurred()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, string(diskDataJSON)) + }) + }) + + Context("VM cloud-init instance ID data", func() { + + BeforeEach(func() { + vmCtx.VM = builder.DummyVirtualMachineA2() + }) + + When("VM cloud-init instance ID already exists in ExtraConfig", func() { + + BeforeEach(func() { + _, err := vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ExtraConfig: []types.BaseOptionValue{ + &types.OptionValue{ + Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, + Value: "ec-instance-id", + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + vmCtx.VM.Annotations = map[string]string{ + vmopv1.InstanceIDAnnotation: "other-instance-id", + } + }) + + It("Should skip backing up the cloud-init instance ID", func() { + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "ec-instance-id") + }) + }) + + When("VM cloud-init instance ID does not exist in ExtraConfig and is set in annotations", func() { + + BeforeEach(func() { + vmCtx.VM.Annotations = map[string]string{ + vmopv1.InstanceIDAnnotation: "annotation-instance-id", + } + vmCtx.VM.UID = "vm-uid" + }) + + It("Should backup the cloud-init instance ID from annotations", func() { + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "annotation-instance-id") + }) + }) + + When("VM cloud-init instance ID does not exist in ExtraConfig and is not set in annotations", func() { + + BeforeEach(func() { + vmCtx.VM.Annotations = nil + vmCtx.VM.UID = "vm-uid" + }) + + It("Should backup the cloud-init instance ID from annotations", func() { + Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "vm-uid") + }) + }) + }) +} + +func verifyBackupDataInExtraConfig( + ctx *builder.TestContextForVCSim, + vcVM *object.VirtualMachine, + expectedKey, expectedValDecoded string) { + + // Get the VM's ExtraConfig and convert it to map. + moID := vcVM.Reference().Value + objVM := ctx.GetVMFromMoID(moID) + Expect(objVM).NotTo(BeNil()) + var moVM mo.VirtualMachine + Expect(objVM.Properties(ctx, objVM.Reference(), []string{"config.extraConfig"}, &moVM)).To(Succeed()) + ecMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig) + + // Verify the expected key exists in ExtraConfig and the decoded values match. + Expect(ecMap).To(HaveKey(expectedKey)) + ecValRaw := ecMap[expectedKey] + ecValDecoded, err := util.TryToDecodeBase64Gzip([]byte(ecValRaw)) + Expect(err).NotTo(HaveOccurred()) + Expect(ecValDecoded).To(Equal(expectedValDecoded)) +} diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go index 12c5c35fd..b78aa26e1 100644 --- a/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/virtualmachine_suite_test.go @@ -15,6 +15,7 @@ func vcSimTests() { Describe("ClusterComputeResource", ccrTests) Describe("Delete", deleteTests) Describe("Publish", publishTests) + Describe("Backup", backupTests) } var suite = builder.NewTestSuite() diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go index 1ef1c92a5..97cf1e489 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go @@ -167,12 +167,6 @@ func (vs *vSphereVMProvider) PublishVirtualMachine( return itemID, nil } -// BackupVirtualMachine backs up the VM data required for restore. -func (vs *vSphereVMProvider) BackupVirtualMachine(ctx goctx.Context, vm *vmopv1.VirtualMachine) error { - // TODO - return nil -} - func (vs *vSphereVMProvider) GetVirtualMachineGuestHeartbeat( ctx goctx.Context, vm *vmopv1.VirtualMachine) (vmopv1.GuestHeartbeatStatus, error) { @@ -364,6 +358,19 @@ func (vs *vSphereVMProvider) updateVirtualMachine( } } + // Back up the VM at the end after a successful update. + if lib.IsVMServiceBackupRestoreFSSEnabled() { + vmCtx.Logger.V(4).Info("Backing up VirtualMachine") + // TODO: Support backing up vAppConfig bootstrap data. + data, _, _, err := GetVirtualMachineBootstrap(vmCtx, vs.k8sClient) + if err != nil { + return err + } + if err := virtualmachine.BackupVirtualMachine(vmCtx, vcVM, data); err != nil { + return err + } + } + return nil } From 3c428b154b5caea84f2da3ee4b1ff7512ee516f5 Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Wed, 18 Oct 2023 18:47:10 -0700 Subject: [PATCH 34/54] Update controller-tools to v0.12.0 (#252) This change updates controller-tools to v0.12.0 to fix issues related to kustomize v5 filling in creationTimeStamps with "null" strings --- ...mware.com_clustervirtualmachineimages.yaml | 3 +- ...or.vmware.com_contentlibraryproviders.yaml | 3 +- ...ator.vmware.com_contentsourcebindings.yaml | 3 +- .../vmoperator.vmware.com_contentsources.yaml | 3 +- ...mware.com_virtualmachineclassbindings.yaml | 3 +- ...ator.vmware.com_virtualmachineclasses.yaml | 3 +- ...rator.vmware.com_virtualmachineimages.yaml | 3 +- ...are.com_virtualmachinepublishrequests.yaml | 3 +- ...vmoperator.vmware.com_virtualmachines.yaml | 3 +- ...tor.vmware.com_virtualmachineservices.yaml | 3 +- ...com_virtualmachinesetresourcepolicies.yaml | 3 +- ....com_virtualmachinewebconsolerequests.yaml | 3 +- ...perator.vmware.com_webconsolerequests.yaml | 3 +- ...ry.vmware.com_clustercontentlibraries.yaml | 3 +- ...vmware.com_clustercontentlibraryitems.yaml | 3 +- ...eregistry.vmware.com_contentlibraries.yaml | 3 +- ....com_contentlibraryitemimportrequests.yaml | 3 +- ...gistry.vmware.com_contentlibraryitems.yaml | 3 +- config/rbac/role.yaml | 1 - config/webhook/manifests.yaml | 2 - hack/tools/go.mod | 27 ++++---- hack/tools/go.sum | 65 ++++++++++++------- 22 files changed, 73 insertions(+), 76 deletions(-) diff --git a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml index f018da591..964090a9e 100644 --- a/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_clustervirtualmachineimages.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: clustervirtualmachineimages.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_contentlibraryproviders.yaml b/config/crd/bases/vmoperator.vmware.com_contentlibraryproviders.yaml index 9687c4aba..482ae6b58 100644 --- a/config/crd/bases/vmoperator.vmware.com_contentlibraryproviders.yaml +++ b/config/crd/bases/vmoperator.vmware.com_contentlibraryproviders.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: contentlibraryproviders.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_contentsourcebindings.yaml b/config/crd/bases/vmoperator.vmware.com_contentsourcebindings.yaml index 175d93a21..2c359fea3 100644 --- a/config/crd/bases/vmoperator.vmware.com_contentsourcebindings.yaml +++ b/config/crd/bases/vmoperator.vmware.com_contentsourcebindings.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: contentsourcebindings.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_contentsources.yaml b/config/crd/bases/vmoperator.vmware.com_contentsources.yaml index c74aa352b..ceac9e9a8 100644 --- a/config/crd/bases/vmoperator.vmware.com_contentsources.yaml +++ b/config/crd/bases/vmoperator.vmware.com_contentsources.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: contentsources.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineclassbindings.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineclassbindings.yaml index 1e9a1246b..3654ad3a1 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineclassbindings.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineclassbindings.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: virtualmachineclassbindings.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineclasses.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineclasses.yaml index 89897f23d..b2e874669 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineclasses.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineclasses.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: virtualmachineclasses.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml index 94e957065..0c9b7f106 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineimages.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: virtualmachineimages.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml index f527493e1..d2c2f78fb 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachinepublishrequests.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: virtualmachinepublishrequests.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml index fb5f31350..5dd11777d 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: virtualmachines.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml index fd956acad..b4020807d 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachineservices.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: virtualmachineservices.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachinesetresourcepolicies.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachinesetresourcepolicies.yaml index ce2e25a8c..1df4884ba 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachinesetresourcepolicies.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachinesetresourcepolicies.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: virtualmachinesetresourcepolicies.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachinewebconsolerequests.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachinewebconsolerequests.yaml index ab8f67baf..5d41c1cce 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachinewebconsolerequests.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachinewebconsolerequests.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: virtualmachinewebconsolerequests.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/bases/vmoperator.vmware.com_webconsolerequests.yaml b/config/crd/bases/vmoperator.vmware.com_webconsolerequests.yaml index cff5a0c69..8b80b06fe 100644 --- a/config/crd/bases/vmoperator.vmware.com_webconsolerequests.yaml +++ b/config/crd/bases/vmoperator.vmware.com_webconsolerequests.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: webconsolerequests.vmoperator.vmware.com spec: group: vmoperator.vmware.com diff --git a/config/crd/external-crds/imageregistry.vmware.com_clustercontentlibraries.yaml b/config/crd/external-crds/imageregistry.vmware.com_clustercontentlibraries.yaml index aece019c2..ad7ce6a32 100644 --- a/config/crd/external-crds/imageregistry.vmware.com_clustercontentlibraries.yaml +++ b/config/crd/external-crds/imageregistry.vmware.com_clustercontentlibraries.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: clustercontentlibraries.imageregistry.vmware.com spec: group: imageregistry.vmware.com diff --git a/config/crd/external-crds/imageregistry.vmware.com_clustercontentlibraryitems.yaml b/config/crd/external-crds/imageregistry.vmware.com_clustercontentlibraryitems.yaml index 633884f8c..d0ebe464a 100644 --- a/config/crd/external-crds/imageregistry.vmware.com_clustercontentlibraryitems.yaml +++ b/config/crd/external-crds/imageregistry.vmware.com_clustercontentlibraryitems.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: clustercontentlibraryitems.imageregistry.vmware.com spec: group: imageregistry.vmware.com diff --git a/config/crd/external-crds/imageregistry.vmware.com_contentlibraries.yaml b/config/crd/external-crds/imageregistry.vmware.com_contentlibraries.yaml index 7e578e3fd..903cf5b36 100644 --- a/config/crd/external-crds/imageregistry.vmware.com_contentlibraries.yaml +++ b/config/crd/external-crds/imageregistry.vmware.com_contentlibraries.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: contentlibraries.imageregistry.vmware.com spec: group: imageregistry.vmware.com diff --git a/config/crd/external-crds/imageregistry.vmware.com_contentlibraryitemimportrequests.yaml b/config/crd/external-crds/imageregistry.vmware.com_contentlibraryitemimportrequests.yaml index 7b20e1ea9..2fc580209 100644 --- a/config/crd/external-crds/imageregistry.vmware.com_contentlibraryitemimportrequests.yaml +++ b/config/crd/external-crds/imageregistry.vmware.com_contentlibraryitemimportrequests.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: contentlibraryitemimportrequests.imageregistry.vmware.com spec: group: imageregistry.vmware.com diff --git a/config/crd/external-crds/imageregistry.vmware.com_contentlibraryitems.yaml b/config/crd/external-crds/imageregistry.vmware.com_contentlibraryitems.yaml index 89f40ab2c..e39c9c696 100644 --- a/config/crd/external-crds/imageregistry.vmware.com_contentlibraryitems.yaml +++ b/config/crd/external-crds/imageregistry.vmware.com_contentlibraryitems.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.12.0 name: contentlibraryitems.imageregistry.vmware.com spec: group: imageregistry.vmware.com diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e5c4e4209..b919ddd78 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -2,7 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: manager-role rules: - apiGroups: diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index d848c383b..20b53d5bc 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -2,7 +2,6 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: - creationTimestamp: null name: mutating-webhook-configuration webhooks: - admissionReviewVersions: @@ -51,7 +50,6 @@ webhooks: apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: - creationTimestamp: null name: validating-webhook-configuration webhooks: - admissionReviewVersions: diff --git a/hack/tools/go.mod b/hack/tools/go.mod index 1a2a25962..e837fd244 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -10,9 +10,9 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad golang.org/x/vuln v1.0.1 - k8s.io/code-generator v0.26.1 + k8s.io/code-generator v0.27.1 sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20220825200008-d242fe21e646 - sigs.k8s.io/controller-tools v0.10.0 + sigs.k8s.io/controller-tools v0.12.0 sigs.k8s.io/kind v0.20.0 sigs.k8s.io/kubebuilder/v3 v3.12.0 sigs.k8s.io/kustomize/kustomize/v5 v5.1.1 @@ -55,10 +55,10 @@ require ( github.com/ettle/strcase v0.1.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fatih/color v1.14.1 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/fatih/structtag v1.2.0 // indirect github.com/firefart/nonamedreturns v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect github.com/go-critic/go-critic v0.6.7 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -91,6 +91,7 @@ require ( github.com/golangci/misspell v0.4.0 // indirect github.com/golangci/revgrep v0.0.0-20220804021717-745bb2f7c2e6 // indirect github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -135,7 +136,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mbilski/exhaustivestruct v1.2.0 // indirect github.com/mgechev/revive v1.2.5 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect @@ -158,10 +159,10 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polyfloyd/go-errorlint v1.1.0 // indirect - github.com/prometheus/client_golang v1.12.1 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/quasilyte/go-ruleguard v0.3.19 // indirect github.com/quasilyte/gogrep v0.5.0 // indirect github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect @@ -225,11 +226,11 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.4.2 // indirect - k8s.io/api v0.25.0 // indirect - k8s.io/apiextensions-apiserver v0.25.0 // indirect - k8s.io/apimachinery v0.25.0 // indirect + k8s.io/api v0.27.1 // indirect + k8s.io/apiextensions-apiserver v0.27.1 // indirect + k8s.io/apimachinery v0.27.1 // indirect k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect - k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961 // indirect k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect mvdan.cc/gofumpt v0.4.0 // indirect diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 2da03ccf1..9f3b26e63 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -128,6 +128,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingaikin/go-header v0.4.3 h1:tEaZKAlqql6SKCY++utLmkPLd6K8IBM20Ha7UVm+mtU= github.com/denis-tingaikin/go-header v0.4.3/go.mod h1:0wOCWuN71D5qIgE2nz9KrKmuYBAC2Mra5RassOIQ2/c= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elastic/crd-ref-docs v0.0.9-0.20220728100728-3a11386f88f1 h1:7G0Et3YwZgR0vsrXWh5kfHzy/zUtfueE2rXtXS/V5As= github.com/elastic/crd-ref-docs v0.0.9-0.20220728100728-3a11386f88f1/go.mod h1:Jd1XDGgrvHzd/+qCf4SwBnOnLl4RaFRjind0ORnHovo= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= @@ -147,8 +148,8 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/firefart/nonamedreturns v1.0.4 h1:abzI1p7mAEPYuR4A+VLKn4eNDOycjYo2phmY9sfv40Y= @@ -157,8 +158,8 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/go-critic/go-critic v0.6.7 h1:1evPrElnLQ2LZtJfmNDzlieDhjnq36SLgNzisx06oPM= @@ -171,9 +172,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -277,6 +280,8 @@ github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSW github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU= @@ -311,8 +316,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI= @@ -447,8 +452,9 @@ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPn github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mbilski/exhaustivestruct v1.2.0 h1:wCBmUnSYufAHO6J4AVWY6ff+oxWxsVFrwgOdMUQePUo= github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc= github.com/mgechev/revive v1.2.5 h1:UF9AR8pOAuwNmhXj2odp4mxv9Nx2qUIwVz8ZsU+Mbec= @@ -524,24 +530,28 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/quasilyte/go-ruleguard v0.3.19 h1:tfMnabXle/HzOb5Xe9CUZYWXKfkS1KwRmZyPmD9nVcc= github.com/quasilyte/go-ruleguard v0.3.19/go.mod h1:lHSn69Scl48I7Gt9cX3VrbsZYvYiBYszZOZW4A+oTEw= github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= @@ -604,6 +614,7 @@ github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YE github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stbenjam/no-sprintf-host-port v0.1.1 h1:tYugd/yrm1O0dV+ThCbaKZh195Dfm07ysF0U6JQXczc= github.com/stbenjam/no-sprintf-host-port v0.1.1/go.mod h1:TLhvtIvONRzdmkFiio4O8LHsN9N74I+PhRquPsxpL0I= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -778,6 +789,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= @@ -795,6 +808,7 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -867,13 +881,14 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1048,6 +1063,7 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1111,6 +1127,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1123,19 +1140,19 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.4.2 h1:6qXr+R5w+ktL5UkwEbPp+fEvfyoMPche6GkOpGHZcLc= honnef.co/go/tools v0.4.2/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= -k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= -k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= -k8s.io/apiextensions-apiserver v0.25.0 h1:CJ9zlyXAbq0FIW8CD7HHyozCMBpDSiH7EdrSTCZcZFY= -k8s.io/apiextensions-apiserver v0.25.0/go.mod h1:3pAjZiN4zw7R8aZC5gR0y3/vCkGlAjCazcg1me8iB/E= -k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= -k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= -k8s.io/code-generator v0.26.1 h1:dusFDsnNSKlMFYhzIM0jAO1OlnTN5WYwQQ+Ai12IIlo= -k8s.io/code-generator v0.26.1/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZmeI3I= +k8s.io/api v0.27.1 h1:Z6zUGQ1Vd10tJ+gHcNNNgkV5emCyW+v2XTmn+CLjSd0= +k8s.io/api v0.27.1/go.mod h1:z5g/BpAiD+f6AArpqNjkY+cji8ueZDU/WV1jcj5Jk4E= +k8s.io/apiextensions-apiserver v0.27.1 h1:Hp7B3KxKHBZ/FxmVFVpaDiXI6CCSr49P1OJjxKO6o4g= +k8s.io/apiextensions-apiserver v0.27.1/go.mod h1:8jEvRDtKjVtWmdkhOqE84EcNWJt/uwF8PC4627UZghY= +k8s.io/apimachinery v0.27.1 h1:EGuZiLI95UQQcClhanryclaQE6xjg1Bts6/L3cD7zyc= +k8s.io/apimachinery v0.27.1/go.mod h1:5ikh59fK3AJ287GUvpUsryoMFtH9zj/ARfWCo3AyXTM= +k8s.io/code-generator v0.27.1 h1:GrfUeUrJ/RtPskIsnChcXOW6h0EGNqty0VxxQ9qYKlM= +k8s.io/code-generator v0.27.1/go.mod h1:iWtpm0ZMG6Gc4daWfITDSIu+WFhFJArYDhj242zcbnY= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08= k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961 h1:pqRVJGQJz6oeZby8qmPKXYIBjyrcv7EHCe/33UkZMYA= k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961/go.mod h1:l8HTwL5fqnlns4jOveW1L75eo7R9KFHxiE0bsPGy428= k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU= @@ -1153,8 +1170,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20220825200008-d242fe21e646 h1:PXyr4ikzBcyW+UmDT3+QNxPmyRgClh9AAELBBeSOJOk= sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20220825200008-d242fe21e646/go.mod h1:nLkMD2WB4Jcix1qfVuJeOF4j5y/VfyeOIlTxG5Wj9co= -sigs.k8s.io/controller-tools v0.10.0 h1:0L5DTDTFB67jm9DkfrONgTGmfc/zYow0ZaHyppizU2U= -sigs.k8s.io/controller-tools v0.10.0/go.mod h1:uvr0EW6IsprfB0jpQq6evtKy+hHyHCXNfdWI5ONPx94= +sigs.k8s.io/controller-tools v0.12.0 h1:TY6CGE6+6hzO7hhJFte65ud3cFmmZW947jajXkuDfBw= +sigs.k8s.io/controller-tools v0.12.0/go.mod h1:rXlpTfFHZMpZA8aGq9ejArgZiieHd+fkk/fTatY8A2M= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kind v0.20.0 h1:f0sc3v9mQbGnjBUaqSFST1dwIuiikKVGgoTwpoP33a8= From aee00744741c5ec9167b65ad46a5003a9ba5aec3 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Thu, 12 Oct 2023 14:53:07 -0500 Subject: [PATCH 35/54] Synthesize v1a1 Prereq condition from v1a2 conditions There is various weirdness around how and what we set the v1a1 Prereq condition to, so this isn't quite a complete match but hopefully good enough. For now just leave the conditions as-is when going from v1a1 to v1a2. We might later want to remove the Prereq - or other v1a1 only conditions - during that conversion. --- api/v1alpha1/virtualmachine_conversion.go | 83 +++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/api/v1alpha1/virtualmachine_conversion.go b/api/v1alpha1/virtualmachine_conversion.go index 536695bd3..39edb9e2f 100644 --- a/api/v1alpha1/virtualmachine_conversion.go +++ b/api/v1alpha1/virtualmachine_conversion.go @@ -541,6 +541,88 @@ func Convert_v1alpha1_VirtualMachineStatus_To_v1alpha2_VirtualMachineStatus( return nil } +func translate_v1alpha2_Conditions_To_v1alpha1_Conditions(conditions []Condition) []Condition { + var preReqCond, vmClassCond, vmImageCond, vmSetResourcePolicy, vmBootstrap *Condition + + for i := range conditions { + c := &conditions[i] + + switch c.Type { + case VirtualMachinePrereqReadyCondition: + preReqCond = c + case v1alpha2.VirtualMachineConditionClassReady: + vmClassCond = c + case v1alpha2.VirtualMachineConditionImageReady: + vmImageCond = c + case v1alpha2.VirtualMachineConditionVMSetResourcePolicyReady: + vmSetResourcePolicy = c + case v1alpha2.VirtualMachineConditionBootstrapReady: + vmBootstrap = c + } + } + + // Try to replicate how the v1a1 provider would set the singular prereqs condition. The class is checked + // first, then the image. Note that the set resource policy and metadata (bootstrap) are not a part of + // the v1a1 prereqs, and are optional. + if vmClassCond != nil && vmClassCond.Status == corev1.ConditionTrue && + vmImageCond != nil && vmImageCond.Status == corev1.ConditionTrue && + (vmSetResourcePolicy == nil || vmSetResourcePolicy.Status == corev1.ConditionTrue) && + (vmBootstrap == nil || vmBootstrap.Status == corev1.ConditionTrue) { + + p := Condition{ + Type: VirtualMachinePrereqReadyCondition, + Status: corev1.ConditionTrue, + } + + if preReqCond != nil { + p.LastTransitionTime = preReqCond.LastTransitionTime + *preReqCond = p + return conditions + } + + p.LastTransitionTime = vmImageCond.LastTransitionTime + return append(conditions, p) + } + + p := Condition{ + Type: VirtualMachinePrereqReadyCondition, + Status: corev1.ConditionFalse, + Severity: ConditionSeverityError, + } + + if vmClassCond != nil && vmClassCond.Status == corev1.ConditionFalse { + p.Reason = VirtualMachineClassNotFoundReason + p.Message = vmClassCond.Message + p.LastTransitionTime = vmClassCond.LastTransitionTime + } else if vmImageCond != nil && vmImageCond.Status == corev1.ConditionFalse { + p.Reason = VirtualMachineImageNotFoundReason + p.Message = vmImageCond.Message + p.LastTransitionTime = vmImageCond.LastTransitionTime + } + + if p.Reason != "" { + if preReqCond != nil { + *preReqCond = p + return conditions + } + + return append(conditions, p) + } + + if vmSetResourcePolicy != nil && vmSetResourcePolicy.Status == corev1.ConditionFalse && + vmBootstrap != nil && vmBootstrap.Status == corev1.ConditionFalse { + + // These are not a part of the v1a1 Prereqs. If either is false, the v1a1 provider would not + // update the prereqs condition, but we don't set the condition to true either until all these + // conditions are true. Just leave things as is to see how strict we really need to be here. + return conditions + } + + // TBD: For now, leave the v1a2 conditions if present since those provide more details. + + return conditions +} + func Convert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus( in *v1alpha2.VirtualMachineStatus, out *VirtualMachineStatus, s apiconversion.Scope) error { @@ -552,6 +634,7 @@ func Convert_v1alpha2_VirtualMachineStatus_To_v1alpha1_VirtualMachineStatus( out.Phase = convert_v1alpha2_Conditions_To_v1alpha1_Phase(in.Conditions) out.VmIp, out.NetworkInterfaces = convert_v1alpha2_NetworkStatus_To_v1alpha1_Network(in.Network) out.LastRestartTime = in.LastRestartTime + out.Conditions = translate_v1alpha2_Conditions_To_v1alpha1_Conditions(out.Conditions) // WARNING: in.Image requires manual conversion: does not exist in peer-type // WARNING: in.Class requires manual conversion: does not exist in peer-type From 25b6933d95656b4178f23a5a7c4229c0cf99573b Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Mon, 23 Oct 2023 12:25:49 -0500 Subject: [PATCH 36/54] Mark v1a2 VirtualMachineClass as namespace scoped The v1a2 class is implicitly scoped because we don't have the bindings. Note this still requires the WCP_Namespaced_VM_Class FFS, and the generated manifest is unchanged because v1a1 is still marked as Cluster scoped. --- api/v1alpha2/virtualmachineclass_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1alpha2/virtualmachineclass_types.go b/api/v1alpha2/virtualmachineclass_types.go index a53500b98..c9ada2031 100644 --- a/api/v1alpha2/virtualmachineclass_types.go +++ b/api/v1alpha2/virtualmachineclass_types.go @@ -244,7 +244,7 @@ type VirtualMachineClassStatus struct { } // +kubebuilder:object:root=true -// +kubebuilder:resource:scope=Cluster,shortName=vmclass +// +kubebuilder:resource:scope=Namespaced,shortName=vmclass // +kubebuilder:storageversion // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="CPU",type="string",JSONPath=".spec.hardware.cpus" From c9f13246c6b9478dc6e8edfae7a8c3a909bcd3af Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Wed, 25 Oct 2023 09:39:37 -0700 Subject: [PATCH 37/54] Enable v1a1 VM mutation webhooks on v1a2 FSS enablement (#256) The v1a1 VM mutating webhooks are still required in a v1a2 environment to preserve fields in v1a1 unknown to v1a2. Since the mutation webhooks also run serially they don't have the problem of racing that a validation webhook would have. --- webhooks/virtualmachine/webhooks.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/webhooks/virtualmachine/webhooks.go b/webhooks/virtualmachine/webhooks.go index 83370a430..041c6d027 100644 --- a/webhooks/virtualmachine/webhooks.go +++ b/webhooks/virtualmachine/webhooks.go @@ -11,15 +11,19 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha1" + v1a1mut "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha1/mutation" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha2" ) func AddToManager(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { if lib.IsVMServiceV1Alpha2FSSEnabled() { - // TODO: We'll likely still need the v1a1 mutation wehbook (at least some limited version of it) if err := v1alpha2.AddToManager(ctx, mgr); err != nil { return errors.Wrap(err, "failed to initialize v1alpha2 webhooks") } + // With v1a2 FSS enabled, the v1a1 VM mutation webhook is added to the manager + if err := v1a1mut.AddToManager(ctx, mgr); err != nil { + return errors.Wrap(err, "failed to initialize v1alpha1 virtual machine mutation webhooks") + } } else { if err := v1alpha1.AddToManager(ctx, mgr); err != nil { return errors.Wrap(err, "failed to initialize v1alpha1 webhooks") From 66510e6fd1c9da7cfef111c00066ca2c7b218049 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Mon, 16 Oct 2023 16:01:26 -0500 Subject: [PATCH 38/54] Various v1a2 bug fixes Accumulated fixes as we've continued to test v1a2 - Add a basic to v1a1 version conversion for SysPrep - Do templating for SysPrep - Fix an off by one preventing from fixing up the MAC address in VDS - Correct API doc copy-paste for the DHCP6 and Gateway6 fields While here, remove some stale test comments vcsim DeployOVF behavior that has been fixed. --- api/v1alpha1/virtualmachine_conversion.go | 6 +++++- api/v1alpha2/virtualmachine_network_types.go | 8 ++++---- .../vmoperator.vmware.com_virtualmachines.yaml | 8 ++++---- .../providers/vsphere2/network/gosc_test.go | 2 +- .../providers/vsphere2/session/session_vm_update.go | 2 +- .../providers/vsphere2/vmlifecycle/bootstrap.go | 5 +---- .../vmlifecycle/bootstrap_linuxprep_test.go | 2 +- .../vsphere2/vmlifecycle/bootstrap_sysprep.go | 6 +++++- .../vsphere2/vmlifecycle/bootstrap_sysprep_test.go | 2 +- .../providers/vsphere2/vmprovider_vm_test.go | 13 +------------ 10 files changed, 24 insertions(+), 30 deletions(-) diff --git a/api/v1alpha1/virtualmachine_conversion.go b/api/v1alpha1/virtualmachine_conversion.go index 39edb9e2f..6885a7ef3 100644 --- a/api/v1alpha1/virtualmachine_conversion.go +++ b/api/v1alpha1/virtualmachine_conversion.go @@ -202,10 +202,11 @@ func convert_v1alpha2_BootstrapSpec_To_v1alpha1_VmMetadata( return nil } + // TODO: v1a2 only has a Secret bootstrap field so that's what we set in v1a1. If this was created + // as v1a1, we need to store the serialized object to know to set either the ConfigMap or Secret field. out := &VirtualMachineMetadata{} if cloudInit := in.CloudInit; cloudInit != nil { - // TODO: Here we don't know if this was originally a Secret or a ConfigMap. out.SecretName = cloudInit.RawCloudConfig.Name switch cloudInit.RawCloudConfig.Key { @@ -214,6 +215,9 @@ func convert_v1alpha2_BootstrapSpec_To_v1alpha1_VmMetadata( case "user-data": out.Transport = VirtualMachineMetadataCloudInitTransport } + } else if sysprep := in.Sysprep; sysprep != nil { + out.SecretName = sysprep.RawSysprep.Name + out.Transport = VirtualMachineMetadataSysprepTransport } else if in.VAppConfig != nil { out.SecretName = in.VAppConfig.RawProperties diff --git a/api/v1alpha2/virtualmachine_network_types.go b/api/v1alpha2/virtualmachine_network_types.go index 745054891..0b59a6af8 100644 --- a/api/v1alpha2/virtualmachine_network_types.go +++ b/api/v1alpha2/virtualmachine_network_types.go @@ -80,7 +80,7 @@ type VirtualMachineNetworkInterfaceSpec struct { // Please note this field is only supported if the network connection // supports DHCP. // - // Please note this field is mutually exclusive with IP4 addresses in the + // Please note this field is mutually exclusive with IP6 addresses in the // Addresses field and the Gateway6 field. // // +optional @@ -109,7 +109,7 @@ type VirtualMachineNetworkInterfaceSpec struct { // supports manual IP allocation. // // If the network connection supports manual IP allocation and the - // Addresses field includes at least one IP4 address, then this field + // Addresses field includes at least one IP6 address, then this field // is required. // // Please note the IP address must include the network prefix length, ex. @@ -257,7 +257,7 @@ type VirtualMachineNetworkSpec struct { // Please note this field is only supported if the network connection // supports DHCP. // - // Please note this field is mutually exclusive with IP4 addresses in the + // Please note this field is mutually exclusive with IP6 addresses in the // Addresses field and the Gateway6 field. // // Please note if the Interfaces field is non-empty then this field is @@ -289,7 +289,7 @@ type VirtualMachineNetworkSpec struct { // supports manual IP allocation. // // If the network connection supports manual IP allocation and the - // Addresses field includes at least one IP4 address, then this field + // Addresses field includes at least one IP6 address, then this field // is required. // // Please note this field is mutually exclusive with DHCP6. diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml index 5dd11777d..12a4f4341 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml @@ -1564,7 +1564,7 @@ spec: description: "DHCP6 indicates whether or not to use DHCP for IP6 networking. \n Please note this field is only supported if the network connection supports DHCP. \n Please note this field - is mutually exclusive with IP4 addresses in the Addresses field + is mutually exclusive with IP6 addresses in the Addresses field and the Gateway6 field. \n Please note if the Interfaces field is non-empty then this field is ignored and should be specified on the elements in the Interfaces list." @@ -1591,7 +1591,7 @@ spec: \n Please note this field is only supported if the network connection supports manual IP allocation. \n If the network connection supports manual IP allocation and the Addresses field includes - at least one IP4 address, then this field is required. \n Please + at least one IP6 address, then this field is required. \n Please note this field is mutually exclusive with DHCP6. \n Please note if the Interfaces field is non-empty then this field is ignored and should be specified on the elements in the Interfaces @@ -1638,7 +1638,7 @@ spec: description: "DHCP6 indicates whether or not this interface uses DHCP for IP6 networking. \n Please note this field is only supported if the network connection supports DHCP. - \n Please note this field is mutually exclusive with IP4 + \n Please note this field is mutually exclusive with IP6 addresses in the Addresses field and the Gateway6 field." type: boolean gateway4: @@ -1656,7 +1656,7 @@ spec: interface. \n Please note this field is only supported if the network connection supports manual IP allocation. \n If the network connection supports manual IP allocation - and the Addresses field includes at least one IP4 address, + and the Addresses field includes at least one IP6 address, then this field is required. \n Please note the IP address must include the network prefix length, ex. 2001:db8:101::1/64. \n Please note this field is mutually exclusive with DHCP6." diff --git a/pkg/vmprovider/providers/vsphere2/network/gosc_test.go b/pkg/vmprovider/providers/vsphere2/network/gosc_test.go index 1eea859a8..9359bf261 100644 --- a/pkg/vmprovider/providers/vsphere2/network/gosc_test.go +++ b/pkg/vmprovider/providers/vsphere2/network/gosc_test.go @@ -16,7 +16,7 @@ import ( var _ = Describe("GOSC", func() { const ( - macAddr1 = "50-8A-80-9D-28-22" + macAddr1 = "50:8A:80:9D:28:22" ipv4Gateway = "192.168.1.1" ipv4 = "192.168.1.10" diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go index baf4e5c74..d82f6ce13 100644 --- a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go @@ -699,7 +699,7 @@ func (s *Session) customize( // with the actual devices. Old code also made this best effort so do that here too. // I've got a larger change that removes the old session stuff, and improves on all this behavior // but I didn't have the BW to sort out all the changes. - if len(updateArgs.NetworkResults.Results) > 1 { + if len(updateArgs.NetworkResults.Results) > 0 { mac := updateArgs.NetworkResults.Results[0].MacAddress if mac == "" { ethCards, _ := resVM.GetNetworkDevices(vmCtx) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go index 1546399e8..8e9f8e289 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go @@ -63,10 +63,7 @@ func DoBootstrap( return err } - if vAppConfig != nil { - // I think the intention was to only apply this to vAppData. Old code would apply it to entire - // Data map but for like SysPrep that data may be base64/gzip'd, and we'd do the template stuff - // prior to plain texting it. + if sysPrep != nil || vAppConfig != nil { bootstrapArgs.TemplateRenderFn = GetTemplateRenderFunc(vmCtx, bootstrapArgs) } diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go index 06fcd4a12..7bd30248a 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_linuxprep_test.go @@ -22,7 +22,7 @@ import ( var _ = Describe("LinuxPrep Bootstrap", func() { const ( - macAddr = "43-AB-B4-1B-7E-87" + macAddr = "43:AB:B4:1B:7E:87" ) var ( diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go index a8fe2baea..dea767cb4 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep.go @@ -24,11 +24,11 @@ func BootstrapSysPrep( bsArgs *BootstrapArgs) (*vimTypes.VirtualMachineConfigSpec, *vimTypes.CustomizationSpec, error) { var data string + key := "unattend" if equality.Semantic.DeepEqual(sysPrepSpec.Sysprep, sysprep.Sysprep{}) { var err error - key := "unattend" if sysPrepSpec.RawSysprep.Key != "" { key = sysPrepSpec.RawSysprep.Key } @@ -48,6 +48,10 @@ func BootstrapSysPrep( return nil, nil, fmt.Errorf("TODO: inlined Sysprep") } + if bsArgs.TemplateRenderFn != nil { + data = bsArgs.TemplateRenderFn(key, data) + } + nicSettingMap, err := network.GuestOSCustomization(bsArgs.NetworkResults) if err != nil { return nil, nil, fmt.Errorf("failed to create GSOC adapter mappings: %w", err) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go index 2a61f0ac7..1aec6e841 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap_sysprep_test.go @@ -22,7 +22,7 @@ import ( var _ = Describe("SysPrep Bootstrap", func() { const ( - macAddr = "43-AB-B4-1B-7E-87" + macAddr = "43:AB:B4:1B:7E:87" ) var ( diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go index fbf325f43..cfb000f8d 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go @@ -167,7 +167,7 @@ func vmTests() { }, ControllerKey: 100, }, - AddressType: string(types.VirtualEthernetCardMacTypeGenerated), + AddressType: string(types.VirtualEthernetCardMacTypeManual), MacAddress: "00:0c:29:93:d7:27", ResourceAllocation: &types.VirtualEthernetCardResourceAllocation{ Reservation: pointer.Int64(42), @@ -370,7 +370,6 @@ func vmTests() { } }) - // FIXME: Has extra NIC b/c of vcsim DeployOVF bug It("Reconfigures the VM with the NIC specified in ConfigSpec", func() { Expect(conditions.IsTrue(vm, vmopv1.VirtualMachineConditionCreated)).To(BeTrue()) @@ -380,17 +379,14 @@ func vmTests() { devList := object.VirtualDeviceList(o.Config.Hardware.Device) l := devList.SelectByType(&types.VirtualEthernetCard{}) Expect(l).To(HaveLen(1)) - // Expect(l).To(HaveLen(1 + 1)) dev := l[0].GetVirtualDevice() - // dev := l[0+1].GetVirtualDevice() backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) Expect(ok).Should(BeTrue()) _, dvpg := getDVPG(ctx, dvpgName) Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) ethDevice, ok := l[0].(*types.VirtualE1000) - // ethDevice, ok := l[0+1].(*types.VirtualE1000) Expect(ok).To(BeTrue()) Expect(ethDevice.AddressType).To(Equal(ethCard.AddressType)) Expect(ethDevice.MacAddress).To(Equal(ethCard.MacAddress)) @@ -413,7 +409,6 @@ func vmTests() { configSpec = &types.VirtualMachineConfigSpec{} }) - // FIXME: Has extra NIC b/c of vcsim DeployOVF bug It("Reconfigures the VM with the default NIC settings from provider", func() { var o mo.VirtualMachine Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) @@ -421,10 +416,8 @@ func vmTests() { devList := object.VirtualDeviceList(o.Config.Hardware.Device) l := devList.SelectByType(&types.VirtualEthernetCard{}) Expect(l).To(HaveLen(1)) - // Expect(l).To(HaveLen(1 + 1)) dev := l[0].GetVirtualDevice() - // dev := l[0+1].GetVirtualDevice() backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) Expect(ok).Should(BeTrue()) _, dvpg := getDVPG(ctx, dvpgName) @@ -547,7 +540,6 @@ func vmTests() { } }) - // FIXME: Has extra NIC b/c of vcsim DeployOVF bug It("Reconfigures the VM with a NIC, GPU and DDPIO device specified in ConfigSpec", func() { var o mo.VirtualMachine Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) @@ -555,17 +547,14 @@ func vmTests() { devList := object.VirtualDeviceList(o.Config.Hardware.Device) l := devList.SelectByType(&types.VirtualEthernetCard{}) Expect(l).To(HaveLen(1)) - // Expect(l).To(HaveLen(1 + 1)) dev := l[0].GetVirtualDevice() - // dev := l[0+1].GetVirtualDevice() backing, ok := dev.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) Expect(ok).Should(BeTrue()) _, dvpg := getDVPG(ctx, dvpgName) Expect(backing.Port.PortgroupKey).To(Equal(dvpg.Reference().Value)) ethDevice, ok := l[0].(*types.VirtualE1000) - // ethDevice, ok := l[0+1].(*types.VirtualE1000) Expect(ok).To(BeTrue()) Expect(ethDevice.AddressType).To(Equal(ethCard.AddressType)) Expect(dev.DeviceInfo).To(Equal(ethCard.VirtualDevice.DeviceInfo)) From 2bec39d9234a5b5c763857dec538a09f7cbabd22 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Thu, 19 Oct 2023 15:10:53 -0500 Subject: [PATCH 39/54] Reduce some clutter in v1a2 Status.Network Try to not show interfaces and IPs that aren't informative. On TKG nodes especially there will a lot of pseudo interface and routes that just make the output hard to read. This is just the first pass and we'll likely need more refinement later. --- .../vsphere2/vmlifecycle/update_status.go | 35 +++++- .../vmlifecycle/update_status_test.go | 113 ++++++++++++++++++ 2 files changed, 142 insertions(+), 6 deletions(-) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go index 62bf35ba4..21dc3c7c4 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go @@ -143,9 +143,9 @@ func getGuestNetworkStatus(guestInfo *types.GuestInfo) *vmopv1.VirtualMachineNet } } - if len(guestInfo.Net) > 0 { - status.Interfaces = make([]vmopv1.VirtualMachineNetworkInterfaceStatus, 0, len(guestInfo.Net)) - for i := range guestInfo.Net { + for i := range guestInfo.Net { + // Skip pseudo devices. + if guestInfo.Net[i].DeviceConfigId != -1 { status.Interfaces = append(status.Interfaces, guestNicInfoToInterfaceStatus(i, &guestInfo.Net[i])) } } @@ -161,7 +161,6 @@ func guestNicInfoToInterfaceStatus(idx int, guestNicInfo *types.GuestNicInfo) vm status := vmopv1.VirtualMachineNetworkInterfaceStatus{} // TODO: What name exactly? The CRD name may be the most useful here but hard to line that up. - // BMV: DeviceConfigId will be -1 for our pseudo-y interfaces. Most likely want to just skip those devices. status.Name = fmt.Sprintf("nic-%d-%d", idx, guestNicInfo.DeviceConfigId) status.IP.MACAddr = guestNicInfo.MacAddress @@ -267,9 +266,33 @@ func convertNetIPRouteConfigInfo(routeConfig *types.NetIpRouteConfigInfo) []vmop return nil } - // TODO: Prob only want to show default routes. Will be very verbose on TKG nodes. - out := make([]vmopv1.VirtualMachineNetworkIPRouteStatus, 0, len(routeConfig.IpRoute)) + // Try to skip routes that are likely not interesting or useful to external users - especially on + // TKG nodes - that would otherwise just clutter the Status output. + skipRoute := func(ipRoute types.NetIpRouteConfigInfoIpRoute) bool { + network, prefix := ipRoute.Network, ipRoute.PrefixLength + + ip := net.ParseIP(network) + if ip == nil { + return true + } + + if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { + return true + } + + if ip.To4() != nil { + return prefix == 32 + } + + return ip.To16() == nil || ip.IsInterfaceLocalMulticast() || ip.IsMulticast() + } + + out := make([]vmopv1.VirtualMachineNetworkIPRouteStatus, 0, 1) for _, ipRoute := range routeConfig.IpRoute { + if skipRoute(ipRoute) { + continue + } + out = append(out, vmopv1.VirtualMachineNetworkIPRouteStatus{ Gateway: vmopv1.VirtualMachineNetworkIPRouteGatewayStatus{ Device: ipRoute.Gateway.Device, diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go index 3d130cdc9..f9050ea97 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go @@ -7,14 +7,127 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" conditions "github.com/vmware-tanzu/vm-operator/pkg/conditions2" + "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/vmlifecycle" + "github.com/vmware-tanzu/vm-operator/test/builder" ) +var _ = Describe("UpdateStatus", func() { + + var ( + ctx *builder.TestContextForVCSim + err error + vmCtx context.VirtualMachineContextA2 + vcVM *object.VirtualMachine + vmMO *mo.VirtualMachine + ) + + BeforeEach(func() { + ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{WithV1A2: true}) + + vm := builder.DummyVirtualMachineA2() + vm.Name = "update-status-test" + + vmCtx = context.VirtualMachineContextA2{ + Context: ctx, + Logger: suite.GetLogger().WithValues("vmName", vm.Name), + VM: vm, + } + + vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + Expect(err).ToNot(HaveOccurred()) + + vmMO = &mo.VirtualMachine{} + }) + + AfterEach(func() { + ctx.AfterEach() + ctx = nil + }) + + JustBeforeEach(func() { + err = vmlifecycle.UpdateStatus(vmCtx, ctx.Client, vcVM, vmMO) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("Network", func() { + + Context("Interfaces", func() { + BeforeEach(func() { + vmMO.Guest = &types.GuestInfo{ + Net: []types.GuestNicInfo{ + { + DeviceConfigId: -1, + MacAddress: "mac-1", + }, + { + DeviceConfigId: 4000, + MacAddress: "mac-4000", + }, + }, + } + }) + + It("Skips pseudo devices", func() { + network := vmCtx.VM.Status.Network + + Expect(network.Interfaces).To(HaveLen(1)) + Expect(network.Interfaces[0].IP.MACAddr).To(Equal("mac-4000")) + }) + }) + + Context("IPRoutes", func() { + BeforeEach(func() { + vmMO.Guest = &types.GuestInfo{ + IpStack: []types.GuestStackInfo{ + { + IpRouteConfig: &types.NetIpRouteConfigInfo{ + IpRoute: []types.NetIpRouteConfigInfoIpRoute{ + { + Network: "192.168.1.0", + PrefixLength: 24, + }, + { + Network: "192.168.1.100", + PrefixLength: 32, + }, + { + Network: "fe80::", + PrefixLength: 64, + }, + { + Network: "ff00::", + PrefixLength: 8, + }, + { + Network: "e9ef:6df5:eb14:42e2:5c09:9982:a9b5:8c2b", + PrefixLength: 48, + }, + }, + }, + }, + }, + } + }) + + It("Skips IPs", func() { + network := vmCtx.VM.Status.Network + + Expect(network.IPRoutes).To(HaveLen(2)) + Expect(network.IPRoutes[0].NetworkAddress).To(Equal("192.168.1.0/24")) + Expect(network.IPRoutes[1].NetworkAddress).To(Equal("e9ef:6df5:eb14:42e2:5c09:9982:a9b5:8c2b/48")) + }) + }) + }) +}) + var _ = Describe("VirtualMachineTools Status to VM Status Condition", func() { Context("markVMToolsRunningStatusCondition", func() { var ( From 5e07995d909b4d7df09ac23aaba4387ebad26a49 Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Fri, 27 Oct 2023 15:26:44 -0700 Subject: [PATCH 40/54] Finalize network v1a2 naming convention (#253) This change checks for v1a1 network name'd interfaces before creating or updating the new ones. Also includes validation webhook changes for verifying that the combined `network-vm-interface` name is a valid DNS1123 label. --- .../providers/vsphere2/network/network.go | 35 ++- .../vsphere2/network/network_test.go | 200 +++++++++++++++++- .../validation/virtualmachine_validator.go | 20 +- .../virtualmachine_validator_unit_test.go | 32 +++ 4 files changed, 274 insertions(+), 13 deletions(-) diff --git a/pkg/vmprovider/providers/vsphere2/network/network.go b/pkg/vmprovider/providers/vsphere2/network/network.go index e5d5d96e5..ed6ca01d7 100644 --- a/pkg/vmprovider/providers/vsphere2/network/network.go +++ b/pkg/vmprovider/providers/vsphere2/network/network.go @@ -16,6 +16,7 @@ import ( "github.com/vmware/govmomi/vim25" vimtypes "github.com/vmware/govmomi/vim25/types" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -261,12 +262,23 @@ func createNetOPNetworkInterface( // If empty, NetOP will try to select a namespace default. networkName := interfaceSpec.Network.Name + netIf := &netopv1alpha1.NetworkInterface{} + netIfKey := types.NamespacedName{ + Namespace: vmCtx.VM.Namespace, + Name: NetOPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, true), + } + + // check if a networkIf object exists with the older (v1a1) naming convention + if err := client.Get(vmCtx, netIfKey, netIf); err != nil { + if !apierrors.IsNotFound(err) { + return nil, err + } - netIf := &netopv1alpha1.NetworkInterface{ - ObjectMeta: metav1.ObjectMeta{ + // if notFound set the netIf to the new v1a2 naming convention + netIf.ObjectMeta = metav1.ObjectMeta{ Name: NetOPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, false), Namespace: vmCtx.VM.Namespace, - }, + } } _, err := controllerutil.CreateOrUpdate(vmCtx, client, netIf, func() error { @@ -404,12 +416,23 @@ func createNCPNetworkInterface( // If empty, NCP will use the namespace default. networkName := interfaceSpec.Network.Name + vnetIf := &ncpv1alpha1.VirtualNetworkInterface{} + vnetIfKey := types.NamespacedName{ + Namespace: vmCtx.VM.Namespace, + Name: NCPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, true), + } + + // check if a networkIf object exists with the older (v1a1) naming convention + if err := client.Get(vmCtx, vnetIfKey, vnetIf); err != nil { + if !apierrors.IsNotFound(err) { + return nil, err + } - vnetIf := &ncpv1alpha1.VirtualNetworkInterface{ - ObjectMeta: metav1.ObjectMeta{ + // if notFound set the vnetIf to use the new v1a2 naming convention + vnetIf.ObjectMeta = metav1.ObjectMeta{ Name: NCPCRName(vmCtx.VM.Name, networkName, interfaceSpec.Name, false), Namespace: vmCtx.VM.Namespace, - }, + } } _, err := controllerutil.CreateOrUpdate(vmCtx, client, vnetIf, func() error { diff --git a/pkg/vmprovider/providers/vsphere2/network/network_test.go b/pkg/vmprovider/providers/vsphere2/network/network_test.go index c57a5e1ab..c8a5c4dce 100644 --- a/pkg/vmprovider/providers/vsphere2/network/network_test.go +++ b/pkg/vmprovider/providers/vsphere2/network/network_test.go @@ -35,8 +35,9 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", func() { vm *vmopv1.VirtualMachine interfaceSpecs []vmopv1.VirtualMachineNetworkInterfaceSpec - results network.NetworkInterfaceResults - err error + results network.NetworkInterfaceResults + err error + initObjects []client.Object ) BeforeEach(func() { @@ -59,7 +60,7 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", func() { }) JustBeforeEach(func() { - ctx = suite.NewTestContextForVCSim(testConfig) + ctx = suite.NewTestContextForVCSim(testConfig, initObjects...) results, err = network.CreateAndWaitForNetworkInterfaces( vmCtx, @@ -73,6 +74,7 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", func() { AfterEach(func() { ctx.AfterEach() ctx = nil + initObjects = nil }) Context("Named Network", func() { @@ -225,6 +227,95 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", func() { Expect(ipConfig.IsIPv4).To(BeFalse()) Expect(ipConfig.Gateway).To(Equal("fd1a:6c85:79fe:7c98:0000:0000:0000:0001")) }) + + When("v1a1 network interface exists", func() { + BeforeEach(func() { + netIf := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, true), + Namespace: vm.Namespace, + }, + Spec: netopv1alpha1.NetworkInterfaceSpec{ + NetworkName: networkName, + Type: netopv1alpha1.NetworkInterfaceTypeVMXNet3, + }, + } + + initObjects = append(initObjects, netIf) + }) + + It("returns success", func() { + // Assert test env is what we expect. + Expect(ctx.NetworkRef.Reference().Type).To(Equal("DistributedVirtualPortgroup")) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate successful NetOP reconcile", func() { + netInterface := &netopv1alpha1.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NetOPCRName(vm.Name, networkName, interfaceName, true), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.NetworkName).To(Equal(networkName)) + + netInterface.Status.NetworkID = ctx.NetworkRef.Reference().Value + netInterface.Status.MacAddress = "" // NetOP doesn't set this. + netInterface.Status.IPConfigs = []netopv1alpha1.IPConfig{ + { + IP: "192.168.1.110", + IPFamily: netopv1alpha1.IPv4Protocol, + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + { + IP: "fd1a:6c85:79fe:7c98:0000:0000:0000:000f", + IPFamily: netopv1alpha1.IPv6Protocol, + Gateway: "fd1a:6c85:79fe:7c98:0000:0000:0000:0001", + SubnetMask: "ffff:ffff:ffff:ff00:0000:0000:0000:0000", + }, + } + netInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.MacAddress).To(BeEmpty()) + Expect(result.ExternalID).To(BeEmpty()) + Expect(result.NetworkID).To(Equal(ctx.NetworkRef.Reference().Value)) + Expect(result.Backing).ToNot(BeNil()) + Expect(result.Backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + Expect(result.Name).To(Equal(interfaceName)) + + Expect(result.IPConfigs).To(HaveLen(2)) + ipConfig := result.IPConfigs[0] + Expect(ipConfig.IPCIDR).To(Equal("192.168.1.110/24")) + Expect(ipConfig.IsIPv4).To(BeTrue()) + Expect(ipConfig.Gateway).To(Equal("192.168.1.1")) + ipConfig = result.IPConfigs[1] + Expect(ipConfig.IPCIDR).To(Equal("fd1a:6c85:79fe:7c98::f/56")) + Expect(ipConfig.IsIPv4).To(BeFalse()) + Expect(ipConfig.Gateway).To(Equal("fd1a:6c85:79fe:7c98:0000:0000:0000:0001")) + }) + }) }) }) @@ -339,6 +430,109 @@ var _ = Describe("CreateAndWaitForNetworkInterfaces", func() { Expect(results.Results[0].Backing).ToNot(BeNil()) Expect(results.Results[0].Backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) }) + + When("v1a1 NCP network interface exists", func() { + BeforeEach(func() { + vnetIf := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NCPCRName(vm.Name, networkName, interfaceName, true), + Namespace: vm.Namespace, + }, + Spec: ncpv1alpha1.VirtualNetworkInterfaceSpec{ + VirtualNetwork: networkName, + }, + } + + initObjects = append(initObjects, vnetIf) + }) + + It("returns success", func() { + // Assert test env is what we expect. + Expect(ctx.NetworkRef.Reference().Type).To(Equal("DistributedVirtualPortgroup")) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network interface is not ready yet")) + Expect(results.Results).To(BeEmpty()) + + By("simulate successful NCP reconcile", func() { + netInterface := &ncpv1alpha1.VirtualNetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Name: network.NCPCRName(vm.Name, networkName, interfaceName, true), + Namespace: vm.Namespace, + }, + } + Expect(ctx.Client.Get(ctx, client.ObjectKeyFromObject(netInterface), netInterface)).To(Succeed()) + Expect(netInterface.Spec.VirtualNetwork).To(Equal(networkName)) + + netInterface.Status.InterfaceID = interfaceID + netInterface.Status.MacAddress = macAddress + netInterface.Status.ProviderStatus = &ncpv1alpha1.VirtualNetworkInterfaceProviderStatus{ + NsxLogicalSwitchID: builder.NsxTLogicalSwitchUUID, + } + netInterface.Status.IPAddresses = []ncpv1alpha1.VirtualNetworkInterfaceIP{ + { + IP: "192.168.1.110", + Gateway: "192.168.1.1", + SubnetMask: "255.255.255.0", + }, + { + IP: "fd1a:6c85:79fe:7c98:0000:0000:0000:000f", + Gateway: "fd1a:6c85:79fe:7c98:0000:0000:0000:0001", + SubnetMask: "ffff:ffff:ffff:ff00:0000:0000:0000:0000", + }, + } + netInterface.Status.Conditions = []ncpv1alpha1.VirtualNetworkCondition{ + { + Type: "Ready", + Status: "True", + }, + } + Expect(ctx.Client.Status().Update(ctx, netInterface)).To(Succeed()) + }) + + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + nil, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + + Expect(results.Results).To(HaveLen(1)) + result := results.Results[0] + Expect(result.MacAddress).To(Equal(macAddress)) + Expect(result.ExternalID).To(Equal(interfaceID)) + Expect(result.NetworkID).To(Equal(builder.NsxTLogicalSwitchUUID)) + Expect(result.Name).To(Equal(interfaceName)) + + Expect(result.IPConfigs).To(HaveLen(2)) + ipConfig := result.IPConfigs[0] + Expect(ipConfig.IPCIDR).To(Equal("192.168.1.110/24")) + Expect(ipConfig.IsIPv4).To(BeTrue()) + Expect(ipConfig.Gateway).To(Equal("192.168.1.1")) + ipConfig = result.IPConfigs[1] + Expect(ipConfig.IPCIDR).To(Equal("fd1a:6c85:79fe:7c98::f/56")) + Expect(ipConfig.IsIPv4).To(BeFalse()) + Expect(ipConfig.Gateway).To(Equal("fd1a:6c85:79fe:7c98:0000:0000:0000:0001")) + + // Without the ClusterMoRef on the first call this will be nil for NSXT. + Expect(result.Backing).To(BeNil()) + + clusterMoRef := ctx.GetSingleClusterCompute().Reference() + results, err = network.CreateAndWaitForNetworkInterfaces( + vmCtx, + ctx.Client, + ctx.VCClient.Client, + ctx.Finder, + &clusterMoRef, + interfaceSpecs) + Expect(err).ToNot(HaveOccurred()) + Expect(results.Results).To(HaveLen(1)) + Expect(results.Results[0].Backing).ToNot(BeNil()) + Expect(results.Results[0].Backing.Reference()).To(Equal(ctx.NetworkRef.Reference())) + }) + }) }) }) }) diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index cc94d9d2e..59b3a9925 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -335,7 +335,7 @@ func (v validator) validateNetwork(ctx *context.WebhookRequestContext, vm *vmopv if defaultNetworkInterface.Name == "" { defaultNetworkInterface.Name = "eth0" } - allErrs = append(allErrs, v.validateNetworkInterfaceSpec(networkPath, defaultNetworkInterface)...) + allErrs = append(allErrs, v.validateNetworkInterfaceSpec(networkPath, defaultNetworkInterface, vm.Name)...) } if len(networkSpec.Interfaces) > 0 { @@ -349,7 +349,7 @@ func (v validator) validateNetwork(ctx *context.WebhookRequestContext, vm *vmopv } for i, interfaceSpec := range networkSpec.Interfaces { - allErrs = append(allErrs, v.validateNetworkInterfaceSpec(p.Index(i), interfaceSpec)...) + allErrs = append(allErrs, v.validateNetworkInterfaceSpec(p.Index(i), interfaceSpec, vm.Name)...) } } @@ -358,11 +358,23 @@ func (v validator) validateNetwork(ctx *context.WebhookRequestContext, vm *vmopv func (v validator) validateNetworkInterfaceSpec( interfacePath *field.Path, - interfaceSpec vmopv1.VirtualMachineNetworkInterfaceSpec) field.ErrorList { + interfaceSpec vmopv1.VirtualMachineNetworkInterfaceSpec, + vmName string) field.ErrorList { var allErrs field.ErrorList + var networkIfCRName string + networkName := interfaceSpec.Network.Name - // TODO: Ensure valid name once we finalize the naming convention for the network interface CRD. + // The networkInterface CR name ("vmName-networkName-interfaceName" or "vmName-interfaceName") needs to be a DNS1123 Label + if networkName != "" { + networkIfCRName = fmt.Sprintf("%s-%s-%s", vmName, networkName, interfaceSpec.Name) + } else { + networkIfCRName = fmt.Sprintf("%s-%s", vmName, interfaceSpec.Name) + } + + for _, msg := range validation.NameIsDNSLabel(networkIfCRName, false) { + allErrs = append(allErrs, field.Invalid(interfacePath.Child("name"), interfaceSpec.Name, msg)) + } var ipv4Addrs, ipv6Addrs []string for i, ipCIDR := range interfaceSpec.Addresses { diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go index 0b650047a..4591ccc5c 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go @@ -594,6 +594,7 @@ func unitTestsValidateCreate() { HostName: "my-vm", Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ { + Name: "eth1", Addresses: []string{ "192.168.1.100/24", "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", @@ -634,6 +635,7 @@ func unitTestsValidateCreate() { Gateway4: "192.168.1.1", Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ { + Name: "eth1", DHCP4: true, }, }, @@ -644,6 +646,36 @@ func unitTestsValidateCreate() { ), }, ), + + Entry("disallow creating VM with network interfaces resulting in a non-DNS1123 combined network interface CR name/label (`vmName-networkName-interfaceName`)", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: fmt.Sprintf("%x", make([]byte, validation.DNS1123LabelMaxLength)), + Network: common.PartialObjectRef{ + Name: "dummy-nw", + }, + }, + { + Name: "dummy_If", + Network: common.PartialObjectRef{ + Name: "dummy-nw", + }, + }, + }, + } + }, + validate: doValidateWithMsg( + fmt.Sprintf(`spec.network.interfaces[0].name: Invalid value: "%x": must be no more than 63 characters`, make([]byte, validation.DNS1123LabelMaxLength)), + `spec.network.interfaces[1].name: Invalid value: "dummy_If": a lowercase RFC 1123 label must consist of lower case alphanumeric characters or '-'`, + `and must start and end with an alphanumeric character (e.g. 'my-name'`, + ` or '123-abc'`, + `regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?')`, + ), + }, + ), ) }) } From dd394de2874eb3cd07a4e591b8cce47559dda60e Mon Sep 17 00:00:00 2001 From: Arunesh Pandey Date: Fri, 27 Oct 2023 16:29:24 -0700 Subject: [PATCH 41/54] Allow privileged users to modify annotations on a VirtualMachine object Currently, we have webhooks to prevent users from modifying system used annotations. This is done by validating if the user making the request is the kube-admin user. However, other parts of the infrastructure (such as WCP service) are not considered privileged, which is required for backup/restore. This changes expands the definition of a privileged account by including WCP service solution user. The code expects this particular environment variable to be present in the pod's environment variable. Testing Done: 1. Create a ServiceAccount 2. Create a ClusterRoleBinding with a roleRef that gives the ServiceAccount permissions to edit/update/delete VM resources 3. Invoke create VM with a reserved annotation ``` $ k apply -f test-yamls/vm.yaml --as system:serviceaccount:default:test-sa Error from server (metadata.annotations.virtualmachine.vmoperator.vmware.com/first-boot-done: Forbidden: modifying this annotation is not allowed for non-admin users): error when creating "test-yamls/vm.yaml": admission webhook "default.validating.virtualmachine.v1alpha1.vmoperator.vmware.com" denied the request: metadata.annotations.virtualmachine.vmoperator.vmware.com/first-boot-done: Forbidden: modifying this annotation is not allowed for non-admin users ``` 5. Modify the VM operator deployment to add the env var. 6. Try to create the VM with the annotation again and verify that it goes through: ``` $ k apply -f test-yamls/vm.yaml --as system:serviceaccount:default:test-sa virtualmachine.vmoperator.vmware.com/parunesh-test-4 created ``` --- .../wcp/vmoperator/manager_env_var_patch.yaml | 6 ++ pkg/builder/auth.go | 11 +++- pkg/builder/mutating_webhook.go | 2 +- pkg/builder/validating_webhook.go | 2 +- pkg/lib/env.go | 24 ++++++++ .../virtualmachine_validator_unit_test.go | 58 +++++++++++++++++-- .../virtualmachine_validator_unit_test.go | 50 +++++++++++++++- 7 files changed, 142 insertions(+), 11 deletions(-) diff --git a/config/wcp/vmoperator/manager_env_var_patch.yaml b/config/wcp/vmoperator/manager_env_var_patch.yaml index ace03dc7b..f45352cd5 100644 --- a/config/wcp/vmoperator/manager_env_var_patch.yaml +++ b/config/wcp/vmoperator/manager_env_var_patch.yaml @@ -81,3 +81,9 @@ value: name: FSS_WCP_VMSERVICE_BACKUPRESTORE value: "" + +- op: add + path: /spec/template/spec/containers/0/env/- + value: + name: PRIVILEGED_USERS + value: "" diff --git a/pkg/builder/auth.go b/pkg/builder/auth.go index 617b485c3..2c71a4eed 100644 --- a/pkg/builder/auth.go +++ b/pkg/builder/auth.go @@ -6,15 +6,24 @@ import ( authv1 "k8s.io/api/authentication/v1" "github.com/vmware-tanzu/vm-operator/pkg/context" + "github.com/vmware-tanzu/vm-operator/pkg/lib" ) -func isPrivilegedAccount(ctx *context.WebhookContext, userInfo authv1.UserInfo) bool { +func IsPrivilegedAccount(ctx *context.WebhookContext, userInfo authv1.UserInfo) bool { username := userInfo.Username if strings.EqualFold(username, kubeAdminUser) { return true } + // Users specified by Pod's environment variable "PRIVILEGED_USERS" are considered privileged. + if lib.IsVMServiceBackupRestoreFSSEnabled() { + privUsers := lib.GetPrivilegedUsers() + if _, ok := privUsers[username]; ok { + return true + } + } + serviceAccount := strings.Join([]string{"system", "serviceaccount", ctx.Namespace, ctx.ServiceAccountName}, ":") return strings.EqualFold(username, serviceAccount) } diff --git a/pkg/builder/mutating_webhook.go b/pkg/builder/mutating_webhook.go index f037746bc..16de1731e 100644 --- a/pkg/builder/mutating_webhook.go +++ b/pkg/builder/mutating_webhook.go @@ -144,7 +144,7 @@ func (h *mutatingWebhookHandler) Handle(_ goctx.Context, req admission.Request) Obj: obj, OldObj: oldObj, UserInfo: req.UserInfo, - IsPrivilegedAccount: isPrivilegedAccount(h.WebhookContext, req.UserInfo), + IsPrivilegedAccount: IsPrivilegedAccount(h.WebhookContext, req.UserInfo), Logger: h.WebhookContext.Logger.WithName(obj.GetNamespace()).WithName(obj.GetName()), } diff --git a/pkg/builder/validating_webhook.go b/pkg/builder/validating_webhook.go index 93ed374f6..b4024f899 100644 --- a/pkg/builder/validating_webhook.go +++ b/pkg/builder/validating_webhook.go @@ -153,7 +153,7 @@ func (h *validatingWebhookHandler) Handle(_ goctx.Context, req admission.Request Obj: obj, OldObj: oldObj, UserInfo: req.UserInfo, - IsPrivilegedAccount: isPrivilegedAccount(h.WebhookContext, req.UserInfo), + IsPrivilegedAccount: IsPrivilegedAccount(h.WebhookContext, req.UserInfo), Logger: h.WebhookContext.Logger.WithName(obj.GetNamespace()).WithName(obj.GetName()), } diff --git a/pkg/lib/env.go b/pkg/lib/env.go index 68059b2c0..33bbdf6e0 100644 --- a/pkg/lib/env.go +++ b/pkg/lib/env.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "strconv" + "strings" "time" "k8s.io/apimachinery/pkg/util/wait" @@ -29,6 +30,13 @@ const ( MaxCreateVMsOnProviderEnv = "MAX_CREATE_VMS_ON_PROVIDER" DefaultMaxCreateVMsOnProvider = 80 + // PrivilegedUsersEnv is the key for the environment variable + // containing a set of privileged users. Currently, users pointed by + // this env variable (if VM Service Backup/Restore FSS is enabled), + // along with kube-admin and system users are allowed to make changes + // to certain restricted annotations on a VirtualMachine resource. + PrivilegedUsersEnv = "PRIVILEGED_USERS" + InstanceStoragePVPlacementFailedTTLEnv = "INSTANCE_STORAGE_PV_PLACEMENT_FAILED_TTL" // DefaultInstanceStoragePVPlacementFailedTTL is the default wait time before declaring PV placement failed // after error annotation is set on PVC. @@ -132,6 +140,22 @@ var IsVMServiceBackupRestoreFSSEnabled = func() bool { return os.Getenv(VMServiceBackupRestoreFSS) == TrueString } +// GetPrivilegedUsers returns a set of privileged users specified as comma +// separated values via the environment variable "PRIVILEGED_USERS". +func GetPrivilegedUsers() map[string]struct{} { + privilegedUsers := make(map[string]struct{}) + + parts := strings.Split(strings.TrimSpace(os.Getenv(PrivilegedUsersEnv)), ",") + for _, part := range parts { + part := strings.TrimSpace(part) + if len(part) > 0 { + privilegedUsers[part] = struct{}{} + } + } + + return privilegedUsers +} + // MaxConcurrentCreateVMsOnProvider returns the percentage of reconciler // threads that can be used to create VMs on the provider concurrently. The // default is 80. diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go index 75bb111b4..525e96cca 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_unit_test.go @@ -25,6 +25,7 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + pkgbuilder "github.com/vmware-tanzu/vm-operator/pkg/builder" "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/topology" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/config" @@ -177,9 +178,10 @@ var errInvalidNetworkProviderTypeNamed = field.Invalid( //nolint:gocyclo func unitTestsValidateCreate() { var ( - ctx *unitValidatingWebhookContext - oldFaultDomainsFunc func() bool - oldImageRegistryFunc func() bool + ctx *unitValidatingWebhookContext + oldFaultDomainsFunc func() bool + oldImageRegistryFunc func() bool + oldVMServiceBackupRestoreFunc func() bool ) const ( @@ -228,6 +230,7 @@ func unitTestsValidateCreate() { powerState vmopv1.VirtualMachinePowerState nextRestartTime string adminOnlyAnnotations bool + isPrivilegedUser bool } validateCreate := func(args createArgs, expectedAllowed bool, expectedReason string, expectedErr error) { @@ -380,6 +383,21 @@ func unitTestsValidateCreate() { ctx.vm.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal } + if args.isPrivilegedUser { + lib.IsVMServiceBackupRestoreFSSEnabled = func() bool { + return true + } + + fakeWCPUser := "sso:wcp-12345-fake-machineid-67890@vsphere.local" + Expect(os.Setenv(lib.PrivilegedUsersEnv, fakeWCPUser)).To(Succeed()) + defer func() { + Expect(os.Unsetenv(lib.PrivilegedUsersEnv)).To(Succeed()) + }() + + ctx.UserInfo.Username = fakeWCPUser + ctx.IsPrivilegedAccount = pkgbuilder.IsPrivilegedAccount(ctx.WebhookContext, ctx.UserInfo) + } + ctx.vm.Spec.PowerState = args.powerState ctx.vm.Spec.NextRestartTime = args.nextRestartTime @@ -413,11 +431,13 @@ func unitTestsValidateCreate() { ctx = newUnitTestContextForValidatingWebhook(false) oldFaultDomainsFunc = lib.IsWcpFaultDomainsFSSEnabled oldImageRegistryFunc = lib.IsWCPVMImageRegistryEnabled + oldVMServiceBackupRestoreFunc = lib.IsVMServiceBackupRestoreFSSEnabled }) AfterEach(func() { lib.IsWcpFaultDomainsFSSEnabled = oldFaultDomainsFunc lib.IsWCPVMImageRegistryEnabled = oldImageRegistryFunc + lib.IsVMServiceBackupRestoreFSSEnabled = oldVMServiceBackupRestoreFunc ctx = nil }) @@ -526,6 +546,8 @@ func unitTestsValidateCreate() { field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), }, ", "), nil), Entry("should allow creating VM with admin-only annotations set by service user", createArgs{isServiceUser: true, adminOnlyAnnotations: true}, true, nil, nil), + + Entry("should allow creating VM with admin-only annotations set by WCP user when the Backup/Restore FSS is enabled", createArgs{adminOnlyAnnotations: true, isPrivilegedUser: true}, true, nil, nil), ) } @@ -625,7 +647,8 @@ func unitTestsValidateUpdateWarnings() { func unitTestsValidateUpdate() { var ( - ctx *unitValidatingWebhookContext + ctx *unitValidatingWebhookContext + oldVMServiceBackupRestoreFunc func() bool ) type updateArgs struct { @@ -650,6 +673,7 @@ func unitTestsValidateUpdate() { addAdminOnlyAnnotations bool updateAdminOnlyAnnotations bool removeAdminOnlyAnnotations bool + isPrivilegedUser bool } validateUpdate := func(args updateArgs, expectedAllowed bool, expectedReason string, expectedErr error) { @@ -713,6 +737,22 @@ func unitTestsValidateUpdate() { ctx.oldVM.Annotations[vmopv1.FirstBootDoneAnnotation] = updateSuffix } + if args.isPrivilegedUser { + lib.IsVMServiceBackupRestoreFSSEnabled = func() bool { + return true + } + + privilegedUsersEnvList := " , foo ,bar , test, " + privilegedUser := "bar" + Expect(os.Setenv(lib.PrivilegedUsersEnv, privilegedUsersEnvList)).To(Succeed()) + defer func() { + Expect(os.Unsetenv(lib.PrivilegedUsersEnv)).To(Succeed()) + }() + + ctx.UserInfo.Username = privilegedUser + ctx.IsPrivilegedAccount = pkgbuilder.IsPrivilegedAccount(ctx.WebhookContext, ctx.UserInfo) + } + // Named network provider undoNamedNetProvider := initNamedNetworkProviderConfig( ctx, @@ -743,9 +783,11 @@ func unitTestsValidateUpdate() { BeforeEach(func() { ctx = newUnitTestContextForValidatingWebhook(true) + oldVMServiceBackupRestoreFunc = lib.IsVMServiceBackupRestoreFSSEnabled }) AfterEach(func() { + lib.IsVMServiceBackupRestoreFSSEnabled = oldVMServiceBackupRestoreFunc ctx = nil }) @@ -822,8 +864,12 @@ func unitTestsValidateUpdate() { field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), }, ", "), nil), Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, addAdminOnlyAnnotations: true}, true, nil, nil), - Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, updateAdminOnlyAnnotations: true}, true, nil, nil), - Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, removeAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow updating admin-only annotations by service user", updateArgs{isServiceUser: true, updateAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow removing admin-only annotations by service user", updateArgs{isServiceUser: true, removeAdminOnlyAnnotations: true}, true, nil, nil), + + Entry("should allow adding admin-only annotations by privileged users", updateArgs{isPrivilegedUser: true, addAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow updating admin-only annotations by privileged users", updateArgs{isPrivilegedUser: true, updateAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow removing admin-only annotations by privileged users", updateArgs{isPrivilegedUser: true, removeAdminOnlyAnnotations: true}, true, nil, nil), ) When("the update is performed while object deletion", func() { diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go index 4591ccc5c..d6c7d4a43 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go @@ -25,6 +25,7 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" + pkgbuilder "github.com/vmware-tanzu/vm-operator/pkg/builder" "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/topology" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/config" @@ -77,7 +78,8 @@ func newUnitTestContextForValidatingWebhook(isUpdate bool) *unitValidatingWebhoo //nolint:gocyclo func unitTestsValidateCreate() { var ( - ctx *unitValidatingWebhookContext + ctx *unitValidatingWebhookContext + oldVMServiceBackupRestoreFunc func() bool ) type createArgs struct { @@ -112,6 +114,7 @@ func unitTestsValidateCreate() { powerState vmopv1.VirtualMachinePowerState nextRestartTime string adminOnlyAnnotations bool + isPrivilegedUser bool } validateCreate := func(args createArgs, expectedAllowed bool, expectedReason string, expectedErr error) { @@ -260,6 +263,21 @@ func unitTestsValidateCreate() { ctx.vm.Annotations[vmopv1.FirstBootDoneAnnotation] = updateSuffix } + if args.isPrivilegedUser { + lib.IsVMServiceBackupRestoreFSSEnabled = func() bool { + return true + } + + fakeWCPUser := "sso:wcp-12345-fake-machineid-67890@vsphere.local" + Expect(os.Setenv(lib.PrivilegedUsersEnv, fakeWCPUser)).To(Succeed()) + defer func() { + Expect(os.Unsetenv(lib.PrivilegedUsersEnv)).To(Succeed()) + }() + + ctx.UserInfo.Username = fakeWCPUser + ctx.IsPrivilegedAccount = pkgbuilder.IsPrivilegedAccount(ctx.WebhookContext, ctx.UserInfo) + } + ctx.vm.Spec.PowerState = args.powerState ctx.vm.Spec.NextRestartTime = args.nextRestartTime @@ -279,11 +297,13 @@ func unitTestsValidateCreate() { BeforeEach(func() { ctx = newUnitTestContextForValidatingWebhook(false) + oldVMServiceBackupRestoreFunc = lib.IsVMServiceBackupRestoreFSSEnabled }) AfterEach(func() { Expect(os.Unsetenv(lib.WcpFaultDomainsFSS)).To(Succeed()) Expect(os.Unsetenv(lib.WindowsSysprepFSS)).To(Succeed()) + lib.IsVMServiceBackupRestoreFSSEnabled = oldVMServiceBackupRestoreFunc ctx = nil }) @@ -380,6 +400,8 @@ func unitTestsValidateCreate() { field.Forbidden(annotationPath.Child(vmopv1.FirstBootDoneAnnotation), "modifying this annotation is not allowed for non-admin users").Error(), }, ", "), nil), Entry("should allow creating VM with admin-only annotations set by service user", createArgs{isServiceUser: true, adminOnlyAnnotations: true}, true, nil, nil), + + Entry("should allow creating VM with admin-only annotations set by WCP user when the Backup/Restore FSS is enabled", createArgs{adminOnlyAnnotations: true, isPrivilegedUser: true}, true, nil, nil), ) Context("Network", func() { @@ -682,7 +704,8 @@ func unitTestsValidateCreate() { func unitTestsValidateUpdate() { var ( - ctx *unitValidatingWebhookContext + ctx *unitValidatingWebhookContext + oldVMServiceBackupRestoreFunc func() bool ) type updateArgs struct { @@ -705,6 +728,7 @@ func unitTestsValidateUpdate() { addAdminOnlyAnnotations bool updateAdminOnlyAnnotations bool removeAdminOnlyAnnotations bool + isPrivilegedUser bool } validateUpdate := func(args updateArgs, expectedAllowed bool, expectedReason string, expectedErr error) { @@ -773,6 +797,22 @@ func unitTestsValidateUpdate() { ctx.oldVM.Annotations[vmopv1.FirstBootDoneAnnotation] = dummyFirstBootDoneVal } + if args.isPrivilegedUser { + lib.IsVMServiceBackupRestoreFSSEnabled = func() bool { + return true + } + + privilegedUsersEnvList := " , foo ,bar , test, " + privilegedUser := "bar" + Expect(os.Setenv(lib.PrivilegedUsersEnv, privilegedUsersEnvList)).To(Succeed()) + defer func() { + Expect(os.Unsetenv(lib.PrivilegedUsersEnv)).To(Succeed()) + }() + + ctx.UserInfo.Username = privilegedUser + ctx.IsPrivilegedAccount = pkgbuilder.IsPrivilegedAccount(ctx.WebhookContext, ctx.UserInfo) + } + ctx.oldVM.Spec.NextRestartTime = args.lastRestartTime ctx.vm.Spec.NextRestartTime = args.nextRestartTime @@ -794,12 +834,14 @@ func unitTestsValidateUpdate() { BeforeEach(func() { ctx = newUnitTestContextForValidatingWebhook(true) + oldVMServiceBackupRestoreFunc = lib.IsVMServiceBackupRestoreFSSEnabled }) AfterEach(func() { ctx = nil Expect(os.Unsetenv(lib.WcpFaultDomainsFSS)).To(Succeed()) Expect(os.Unsetenv(lib.WindowsSysprepFSS)).To(Succeed()) + lib.IsVMServiceBackupRestoreFSSEnabled = oldVMServiceBackupRestoreFunc }) msg := "field is immutable" @@ -870,6 +912,10 @@ func unitTestsValidateUpdate() { Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, addAdminOnlyAnnotations: true}, true, nil, nil), Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, updateAdminOnlyAnnotations: true}, true, nil, nil), Entry("should allow adding admin-only annotations by service user", updateArgs{isServiceUser: true, removeAdminOnlyAnnotations: true}, true, nil, nil), + + Entry("should allow adding admin-only annotations by privileged users", updateArgs{isPrivilegedUser: true, addAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow updating admin-only annotations by privileged users", updateArgs{isPrivilegedUser: true, updateAdminOnlyAnnotations: true}, true, nil, nil), + Entry("should allow removing admin-only annotations by privileged users", updateArgs{isPrivilegedUser: true, removeAdminOnlyAnnotations: true}, true, nil, nil), ) When("the update is performed while object deletion", func() { From 662595b9c9087a6fdd165e5c5048b8d7c1137e81 Mon Sep 17 00:00:00 2001 From: akutz Date: Tue, 31 Oct 2023 09:40:33 -0500 Subject: [PATCH 42/54] Addressing dependabot alerts This patch addresses the following dependabot alerts: * https://github.com/vmware-tanzu/vm-operator/security/dependabot/32 * https://github.com/vmware-tanzu/vm-operator/security/dependabot/33 * https://github.com/vmware-tanzu/vm-operator/security/dependabot/34 * https://github.com/vmware-tanzu/vm-operator/security/dependabot/35 * https://github.com/vmware-tanzu/vm-operator/security/dependabot/36 * https://github.com/vmware-tanzu/vm-operator/security/dependabot/37 --- api/go.mod | 5 +++-- go.mod | 10 +++++----- go.sum | 8 ++++---- hack/tools/go.mod | 8 +++++--- hack/tools/go.sum | 12 ++++++------ 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/api/go.mod b/api/go.mod index 39bacb004..ea2acd98a 100644 --- a/api/go.mod +++ b/api/go.mod @@ -56,11 +56,12 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - // per the following dependabot alerts: // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/20 // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/21 + // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/32 + // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/34 + // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/36 golang.org/x/net v0.7.0 // indirect - // per the following dependabot alerts: // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/22 golang.org/x/text v0.7.0 // indirect; per gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.mod b/go.mod index 864d7648c..dacd67c48 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ replace ( require ( github.com/davecgh/go-spew v1.1.1 - github.com/envoyproxy/go-control-plane v0.10.3 + github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f github.com/go-logr/logr v1.2.4 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.1 @@ -23,12 +23,12 @@ require ( github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-00010101000000-000000000000 github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-00010101000000-000000000000 github.com/vmware/govmomi v0.31.0 - // per the following dependabot alerts: // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/24 golang.org/x/net v0.17.0 // indirect golang.org/x/text v0.13.0 gomodules.xyz/jsonpatch/v2 v2.4.0 - google.golang.org/grpc v1.54.0 + // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/37 + google.golang.org/grpc v1.56.3 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.28.0 k8s.io/apiextensions-apiserver v0.28.0 @@ -44,9 +44,9 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect - github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b // indirect + github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/envoyproxy/protoc-gen-validate v0.9.1 // indirect + github.com/envoyproxy/protoc-gen-validate v0.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect diff --git a/go.sum b/go.sum index 4d5dd7bbb..0a2d246b2 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b h1:ACGZRIr7HsgBKHsueQ1yM4WaVaXh21ynwqsF8M8tXhA= -github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -755,8 +755,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= -google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/hack/tools/go.mod b/hack/tools/go.mod index e837fd244..bf1e0b448 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -209,13 +209,15 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect - golang.org/x/crypto v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect + // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/33 + // * https://github.com/vmware-tanzu/vm-operator/security/dependabot/35 + golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.11.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 9f3b26e63..9923c02f1 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -702,8 +702,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -796,8 +796,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -893,8 +893,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From a4d06bd322c9eeda6ee2751c09118b15d9a65eac Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Tue, 31 Oct 2023 19:59:39 -0400 Subject: [PATCH 43/54] =?UTF-8?q?=E2=9C=A8=20Persist=20PVC=20related=20inf?= =?UTF-8?q?o=20in=20VM=20ExtraConfig=20for=20backup=20(#257)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR stores a VM's disk UUID to PVC mapping info in ExtraConfig during the backup process. This will enable us to register the volumes of the VM with the same PVC name and access modes during the restore. The change has also been added to the v1a2 package. A helper function is added in vmprovider_vm_utils.go to retrieve the disk UUID to PVC mapping. This eliminates the need to pass a K8s client in the backup.go file. Also, the BackupVirtualMachine function is updated to have all the necessary params from the new BackupOptions struct. --- pkg/context/backupvirtualmachine_context.go | 38 ++++++ .../vsphere/virtualmachine/backup.go | 82 ++++++------ .../vsphere/virtualmachine/backup_test.go | 122 ++++++++++++++---- .../providers/vsphere/vmprovider_vm.go | 16 ++- .../providers/vsphere/vmprovider_vm_utils.go | 40 ++++++ .../vsphere/vmprovider_vm_utils_test.go | 65 ++++++++++ .../vsphere2/virtualmachine/backup.go | 82 ++++++------ .../vsphere2/virtualmachine/backup_test.go | 120 ++++++++++++++--- .../providers/vsphere2/vmprovider_vm.go | 16 ++- .../providers/vsphere2/vmprovider_vm_utils.go | 40 ++++++ .../vsphere2/vmprovider_vm_utils_test.go | 69 ++++++++++ 11 files changed, 567 insertions(+), 123 deletions(-) create mode 100644 pkg/context/backupvirtualmachine_context.go diff --git a/pkg/context/backupvirtualmachine_context.go b/pkg/context/backupvirtualmachine_context.go new file mode 100644 index 000000000..b703864e9 --- /dev/null +++ b/pkg/context/backupvirtualmachine_context.go @@ -0,0 +1,38 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package context + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + + "github.com/vmware/govmomi/object" +) + +// BackupVirtualMachineContext is the context used for storing backup data of VM +// and its related objects. +type BackupVirtualMachineContext struct { + VMCtx VirtualMachineContext + VcVM *object.VirtualMachine + BootstrapData map[string]string + DiskUUIDToPVC map[string]corev1.PersistentVolumeClaim +} + +func (c *BackupVirtualMachineContext) String() string { + return fmt.Sprintf("Backup %s", c.VMCtx.String()) +} + +// BackupVirtualMachineContextA2 is the context used for storing backup data of +// VM and its related objects. +type BackupVirtualMachineContextA2 struct { + VMCtx VirtualMachineContextA2 + VcVM *object.VirtualMachine + BootstrapData map[string]string + DiskUUIDToPVC map[string]corev1.PersistentVolumeClaim +} + +func (c *BackupVirtualMachineContextA2) String() string { + return fmt.Sprintf("Backup %s", c.VMCtx.String()) +} diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go b/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go index 1ce978917..8ef722932 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go @@ -4,12 +4,11 @@ package virtualmachine import ( - goctx "context" "encoding/json" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/yaml" - "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" @@ -20,38 +19,38 @@ import ( ) type VMDiskData struct { - // ID of the virtual disk object (only set for FCDs). - VDiskID string // Filename contains the datastore path to the virtual disk. FileName string + // PVCName is the name of the PVC backed by the virtual disk. + PVCName string + // AccessMode is the access modes of the PVC backed by the virtual disk. + AccessModes []corev1.PersistentVolumeAccessMode } // BackupVirtualMachine backs up the required data of a VM into its ExtraConfig. // Currently, the following data is backed up: // - Kubernetes VirtualMachine object in YAML format (without its .status field). // - VM bootstrap data in JSON (if provided). -// - List of VM disk data in JSON (including FCDs attached to the VM). -func BackupVirtualMachine( - vmCtx context.VirtualMachineContext, - vcVM *object.VirtualMachine, - bootstrapData map[string]string) error { +// - VM disk data in JSON (if created and attached by PVCs). +func BackupVirtualMachine(ctx context.BackupVirtualMachineContext) error { var moVM mo.VirtualMachine - if err := vcVM.Properties(vmCtx, vcVM.Reference(), + if err := ctx.VcVM.Properties(ctx.VMCtx, ctx.VcVM.Reference(), []string{"config.extraConfig"}, &moVM); err != nil { - vmCtx.Logger.Error(err, "Failed to get VM properties for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get VM properties for backup") return err } curEcMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig) var ecToUpdate []types.BaseOptionValue - vmKubeDataBackup, err := getDesiredVMKubeDataForBackup(vmCtx.VM, curEcMap) + vmKubeDataBackup, err := getDesiredVMKubeDataForBackup(ctx.VMCtx.VM, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get VM kube data for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get VM kube data for backup") return err } + if vmKubeDataBackup == "" { - vmCtx.Logger.V(4).Info("Skipping VM kube data backup as unchanged") + ctx.VMCtx.Logger.V(4).Info("Skipping VM kube data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ Key: constants.BackupVMKubeDataExtraConfigKey, @@ -59,13 +58,14 @@ func BackupVirtualMachine( }) } - instanceIDBackup, err := getDesiredCloudInitInstanceIDForBackup(vmCtx.VM, curEcMap) + instanceIDBackup, err := getDesiredCloudInitInstanceIDForBackup(ctx.VMCtx.VM, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get cloud-init instance ID for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get cloud-init instance ID for backup") return err } + if instanceIDBackup == "" { - vmCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored") + ctx.VMCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, @@ -73,13 +73,14 @@ func BackupVirtualMachine( }) } - bootstrapDataBackup, err := getDesiredBootstrapDataForBackup(bootstrapData, curEcMap) + bootstrapDataBackup, err := getDesiredBootstrapDataForBackup(ctx.BootstrapData, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get VM bootstrap data for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get VM bootstrap data for backup") return err } + if bootstrapDataBackup == "" { - vmCtx.Logger.V(4).Info("Skipping VM bootstrap data backup as unchanged") + ctx.VMCtx.Logger.V(4).Info("Skipping VM bootstrap data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ Key: constants.BackupVMBootstrapDataExtraConfigKey, @@ -87,13 +88,14 @@ func BackupVirtualMachine( }) } - diskDataBackup, err := getDesiredDiskDataForBackup(vmCtx, vcVM, curEcMap) + diskDataBackup, err := getDesiredDiskDataForBackup(ctx, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get VM disk data for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get VM disk data for backup") return err } + if diskDataBackup == "" { - vmCtx.Logger.V(4).Info("Skipping VM disk data backup as unchanged") + ctx.VMCtx.Logger.V(4).Info("Skipping VM disk data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ Key: constants.BackupVMDiskDataExtraConfigKey, @@ -102,12 +104,12 @@ func BackupVirtualMachine( } if len(ecToUpdate) != 0 { - vmCtx.Logger.Info("Updating VM ExtraConfig with backup data") - vmCtx.Logger.V(4).Info("", "ExtraConfig", ecToUpdate) - if _, err := vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ctx.VMCtx.Logger.Info("Updating VM ExtraConfig with backup data") + ctx.VMCtx.Logger.V(4).Info("", "ExtraConfig", ecToUpdate) + if _, err := ctx.VcVM.Reconfigure(ctx.VMCtx, types.VirtualMachineConfigSpec{ ExtraConfig: ecToUpdate, }); err != nil { - vmCtx.Logger.Error(err, "Failed to update VM ExtraConfig for backup") + ctx.VMCtx.Logger.Error(err, "Failed to update VM ExtraConfig for backup") return err } } @@ -194,10 +196,14 @@ func getDesiredBootstrapDataForBackup( } func getDesiredDiskDataForBackup( - ctx goctx.Context, - vcVM *object.VirtualMachine, + ctx context.BackupVirtualMachineContext, ecMap map[string]string) (string, error) { - deviceList, err := vcVM.Device(ctx) + // Return an empty string to skip backup if no disk uuid to PVC is specified. + if len(ctx.DiskUUIDToPVC) == 0 { + return "", nil + } + + deviceList, err := ctx.VcVM.Device(ctx.VMCtx) if err != nil { return "", err } @@ -205,16 +211,14 @@ func getDesiredDiskDataForBackup( var diskData []VMDiskData for _, device := range deviceList.SelectByType((*types.VirtualDisk)(nil)) { if disk, ok := device.(*types.VirtualDisk); ok { - vmDiskData := VMDiskData{} - if disk.VDiskId != nil { - vmDiskData.VDiskID = disk.VDiskId.Id - } if b, ok := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok { - vmDiskData.FileName = b.FileName - } - // Only add the disk data if it's not empty. - if vmDiskData != (VMDiskData{}) { - diskData = append(diskData, vmDiskData) + if pvc, ok := ctx.DiskUUIDToPVC[b.Uuid]; ok { + diskData = append(diskData, VMDiskData{ + FileName: b.FileName, + PVCName: pvc.Name, + AccessModes: pvc.Spec.AccessModes, + }) + } } } } diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go b/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go index 13a0aad9c..fbe47f966 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go @@ -6,6 +6,7 @@ package virtualmachine_test import ( "encoding/json" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/yaml" . "github.com/onsi/ginkgo" @@ -23,6 +24,14 @@ import ( ) func backupTests() { + const ( + // These are the default values of the vcsim VM that are used to + // construct the expected backup data in the following tests. + vcSimVMPath = "DC0_C0_RP0_VM0" + vcSimDiskUUID = "be8d2471-f32e-5c7e-a89b-22cb8e533890" + vcSimDiskFileName = "[LocalDS_0] DC0_C0_RP0_VM0/disk1.vmdk" + ) + var ( ctx *builder.TestContextForVCSim vcVM *object.VirtualMachine @@ -33,7 +42,7 @@ func backupTests() { ctx = suite.NewTestContextForVCSim(builder.VCSimTestConfig{}) var err error - vcVM, err = ctx.Finder.VirtualMachine(ctx, "DC0_C0_RP0_VM0") + vcVM, err = ctx.Finder.VirtualMachine(ctx, vcSimVMPath) Expect(err).NotTo(HaveOccurred()) vmCtx = context.VirtualMachineContext{ @@ -76,7 +85,13 @@ func backupTests() { It("Should backup VM kube data YAML with the latest spec", func() { vmCtx.VM.ObjectMeta.Generation = 2 - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + backupVMCtx := context.BackupVirtualMachineContext{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) vmCopy := vmCtx.VM.DeepCopy() vmCopy.Status = vmopv1.VirtualMachineStatus{} @@ -112,7 +127,13 @@ func backupTests() { It("Should skip backing up VM kube data", func() { // Update the VM to verify its kube data is not backed up in ExtraConfig. vmCtx.VM.Labels = map[string]string{"foo": "bar"} - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + backupVMCtx := context.BackupVirtualMachineContext{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, kubeDataBackup) }) }) @@ -122,7 +143,13 @@ func backupTests() { It("Should back up bootstrap data as JSON in ExtraConfig", func() { bootstrapDataRaw := map[string]string{"foo": "bar"} - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, bootstrapDataRaw)).To(Succeed()) + backupVMCtx := context.BackupVirtualMachineContext{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: bootstrapDataRaw, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) bootstrapDataJSON, err := json.Marshal(bootstrapDataRaw) Expect(err).NotTo(HaveOccurred()) @@ -132,19 +159,47 @@ func backupTests() { Context("VM Disk data", func() { - It("Should backup VM disk data as JSON in ExtraConfig", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + When("VM has no disks that are attached from PVCs", func() { - // Use the default disk info from the vcSim VM for testing. - diskData := []virtualmachine.VMDiskData{ - { - VDiskID: "", - FileName: "[LocalDS_0] DC0_C0_RP0_VM0/disk1.vmdk", - }, - } - diskDataJSON, err := json.Marshal(diskData) - Expect(err).NotTo(HaveOccurred()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, string(diskDataJSON)) + It("Should skip backing up VM disk data", func() { + backupVMCtx := context.BackupVirtualMachineContext{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, "") + }) + }) + + When("VM has disks that are attached from PVCs", func() { + + It("Should backup VM disk data as JSON in ExtraConfig", func() { + dummyPVC := builder.DummyPersistentVolumeClaim() + diskUUIDToPVC := map[string]corev1.PersistentVolumeClaim{ + vcSimDiskUUID: *dummyPVC, + } + + backupVMCtx := context.BackupVirtualMachineContext{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: diskUUIDToPVC, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) + + diskData := []virtualmachine.VMDiskData{ + { + FileName: vcSimDiskFileName, + PVCName: dummyPVC.Name, + AccessModes: dummyPVC.Spec.AccessModes, + }, + } + diskDataJSON, err := json.Marshal(diskData) + Expect(err).NotTo(HaveOccurred()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, string(diskDataJSON)) + }) }) }) @@ -171,8 +226,14 @@ func backupTests() { } }) - It("Should skip backing up the cloud-init instance ID", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + It("Should not change the cloud-init instance ID in VM's ExtraConfig", func() { + backupVMCtx := context.BackupVirtualMachineContext{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "ec-instance-id") }) }) @@ -187,7 +248,13 @@ func backupTests() { }) It("Should backup the cloud-init instance ID from annotations", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + backupVMCtx := context.BackupVirtualMachineContext{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "annotation-instance-id") }) }) @@ -199,9 +266,14 @@ func backupTests() { vmCtx.VM.UID = "vm-uid" }) - It("Should backup the cloud-init instance ID from annotations", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "vm-uid") + It("Should backup the cloud-init instance ID from VM K8s resource UID", func() { + backupVMCtx := context.BackupVirtualMachineContext{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) }) }) }) @@ -220,6 +292,12 @@ func verifyBackupDataInExtraConfig( Expect(objVM.Properties(ctx, objVM.Reference(), []string{"config.extraConfig"}, &moVM)).To(Succeed()) ecMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig) + // Verify the expected key doesn't exist in ExtraConfig if the expected value is empty. + if expectedValDecoded == "" { + Expect(ecMap).NotTo(HaveKey(expectedKey)) + return + } + // Verify the expected key exists in ExtraConfig and the decoded values match. Expect(ecMap).To(HaveKey(expectedKey)) ecValRaw := ecMap[expectedKey] diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm.go index 4a012cdf6..a1a4f9bf3 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm.go @@ -321,9 +321,23 @@ func (vs *vSphereVMProvider) updateVirtualMachine( vmCtx.Logger.V(4).Info("Backing up VirtualMachine") data, err := GetVMMetadata(vmCtx, vs.k8sClient) if err != nil { + vmCtx.Logger.Error(err, "Failed to get VM's metadata for backup") return err } - if err := virtualmachine.BackupVirtualMachine(vmCtx, vcVM, data.Data); err != nil { + diskUUIDToPVC, err := GetAttachedDiskUUIDToPVC(vmCtx, vs.k8sClient) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get VM's attached PVCs for backup") + return err + } + + backupVMCtx := context.BackupVirtualMachineContext{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: data.Data, + DiskUUIDToPVC: diskUUIDToPVC, + } + if err := virtualmachine.BackupVirtualMachine(backupVMCtx); err != nil { + vmCtx.Logger.Error(err, "Failed to back up VM") return err } } diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm_utils.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm_utils.go index b89b1b930..ac0c52e46 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm_utils.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm_utils.go @@ -472,3 +472,43 @@ func HardwareVersionForPVCandPCIDevices(imageHWVersion int32, configSpec *types. return configSpecHWVersion } + +// GetAttachedDiskUUIDToPVC returns a map of disk UUID to PVC object for all +// attached disks by checking the VM's spec and status of volumes. +func GetAttachedDiskUUIDToPVC( + vmCtx context.VirtualMachineContext, + k8sClient ctrlclient.Client) (map[string]corev1.PersistentVolumeClaim, error) { + if !HasPVC(vmCtx.VM.Spec) { + return nil, nil + } + + vmVolNameToPVCName := map[string]string{} + for _, vol := range vmCtx.VM.Spec.Volumes { + if pvc := vol.PersistentVolumeClaim; pvc != nil { + vmVolNameToPVCName[vol.Name] = pvc.ClaimName + } + } + + diskUUIDToPVC := map[string]corev1.PersistentVolumeClaim{} + for _, vol := range vmCtx.VM.Status.Volumes { + if !vol.Attached || vol.DiskUuid == "" { + continue + } + + pvcName := vmVolNameToPVCName[vol.Name] + // This could happen if the volume was just removed from VM spec but not reconciled yet. + if pvcName == "" { + continue + } + + pvcObj := corev1.PersistentVolumeClaim{} + objKey := ctrlclient.ObjectKey{Name: pvcName, Namespace: vmCtx.VM.Namespace} + if err := k8sClient.Get(vmCtx, objKey, &pvcObj); err != nil { + return nil, err + } + + diskUUIDToPVC[vol.DiskUuid] = pvcObj + } + + return diskUUIDToPVC, nil +} diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm_utils_test.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm_utils_test.go index b01db2177..32d00fc4f 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm_utils_test.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm_utils_test.go @@ -871,4 +871,69 @@ func vmUtilTests() { Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(16))) }) }) + + Context("GetAttachedDiskUuidToPVC", func() { + const ( + attachedDiskUUID = "dummy-uuid" + ) + var ( + attachedPVC *corev1.PersistentVolumeClaim + ) + + BeforeEach(func() { + // Create multiple PVCs to verity only the expected one is returned. + unusedPVC := builder.DummyPersistentVolumeClaim() + unusedPVC.Name = "unused-pvc" + unusedPVC.Namespace = vmCtx.VM.Namespace + unattachedPVC := builder.DummyPersistentVolumeClaim() + unattachedPVC.Name = "unattached-pvc" + unattachedPVC.Namespace = vmCtx.VM.Namespace + attachedPVC = builder.DummyPersistentVolumeClaim() + attachedPVC.Name = "attached-pvc" + attachedPVC.Namespace = vmCtx.VM.Namespace + initObjects = append(initObjects, unusedPVC, unattachedPVC, attachedPVC) + + vmCtx.VM.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: "unattached-vol", + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: unattachedPVC.Name, + }, + }, + }, + { + Name: "attached-vol", + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: attachedPVC.Name, + }, + }, + }, + } + + vmCtx.VM.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: "unattached-vol", + Attached: false, + DiskUuid: "unattached-disk-uuid", + }, + { + Name: "attached-vol", + Attached: true, + DiskUuid: attachedDiskUUID, + }, + } + }) + + It("Should return a map of disk uuid to PVCs that are attached to the VM", func() { + diskUUIDToPVCMap, err := vsphere.GetAttachedDiskUUIDToPVC(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(diskUUIDToPVCMap).To(HaveLen(1)) + Expect(diskUUIDToPVCMap).To(HaveKey(attachedDiskUUID)) + pvc := diskUUIDToPVCMap[attachedDiskUUID] + Expect(pvc.Name).To(Equal(attachedPVC.Name)) + Expect(pvc.Namespace).To(Equal(attachedPVC.Namespace)) + }) + }) } diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go index bf40f2181..2f5f3b7a9 100644 --- a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go @@ -4,12 +4,11 @@ package virtualmachine import ( - goctx "context" "encoding/json" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/yaml" - "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" @@ -20,38 +19,38 @@ import ( ) type VMDiskData struct { - // ID of the virtual disk object (only set for FCDs). - VDiskID string // Filename contains the datastore path to the virtual disk. FileName string + // PVCName is the name of the PVC backed by the virtual disk. + PVCName string + // AccessMode is the access modes of the PVC backed by the virtual disk. + AccessModes []corev1.PersistentVolumeAccessMode } // BackupVirtualMachine backs up the required data of a VM into its ExtraConfig. // Currently, the following data is backed up: // - Kubernetes VirtualMachine object in YAML format (without its .status field). // - VM bootstrap data in JSON (if provided). -// - List of VM disk data in JSON (including FCDs attached to the VM). -func BackupVirtualMachine( - vmCtx context.VirtualMachineContextA2, - vcVM *object.VirtualMachine, - bootstrapData map[string]string) error { +// - VM disk data in JSON (if created and attached by PVCs). +func BackupVirtualMachine(ctx context.BackupVirtualMachineContextA2) error { var moVM mo.VirtualMachine - if err := vcVM.Properties(vmCtx, vcVM.Reference(), + if err := ctx.VcVM.Properties(ctx.VMCtx, ctx.VcVM.Reference(), []string{"config.extraConfig"}, &moVM); err != nil { - vmCtx.Logger.Error(err, "Failed to get VM properties for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get VM properties for backup") return err } curEcMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig) var ecToUpdate []types.BaseOptionValue - vmKubeDataBackup, err := getDesiredVMKubeDataForBackup(vmCtx.VM, curEcMap) + vmKubeDataBackup, err := getDesiredVMKubeDataForBackup(ctx.VMCtx.VM, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get VM kube data for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get VM kube data for backup") return err } + if vmKubeDataBackup == "" { - vmCtx.Logger.V(4).Info("Skipping VM kube data backup as unchanged") + ctx.VMCtx.Logger.V(4).Info("Skipping VM kube data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ Key: constants.BackupVMKubeDataExtraConfigKey, @@ -59,13 +58,14 @@ func BackupVirtualMachine( }) } - instanceIDBackup, err := getDesiredCloudInitInstanceIDForBackup(vmCtx.VM, curEcMap) + instanceIDBackup, err := getDesiredCloudInitInstanceIDForBackup(ctx.VMCtx.VM, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get cloud-init instance ID for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get cloud-init instance ID for backup") return err } + if instanceIDBackup == "" { - vmCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored") + ctx.VMCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, @@ -73,13 +73,14 @@ func BackupVirtualMachine( }) } - bootstrapDataBackup, err := getDesiredBootstrapDataForBackup(bootstrapData, curEcMap) + bootstrapDataBackup, err := getDesiredBootstrapDataForBackup(ctx.BootstrapData, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get VM bootstrap data for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get VM bootstrap data for backup") return err } + if bootstrapDataBackup == "" { - vmCtx.Logger.V(4).Info("Skipping VM bootstrap data backup as unchanged") + ctx.VMCtx.Logger.V(4).Info("Skipping VM bootstrap data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ Key: constants.BackupVMBootstrapDataExtraConfigKey, @@ -87,13 +88,14 @@ func BackupVirtualMachine( }) } - diskDataBackup, err := getDesiredDiskDataForBackup(vmCtx, vcVM, curEcMap) + diskDataBackup, err := getDesiredDiskDataForBackup(ctx, curEcMap) if err != nil { - vmCtx.Logger.Error(err, "Failed to get VM disk data for backup") + ctx.VMCtx.Logger.Error(err, "Failed to get VM disk data for backup") return err } + if diskDataBackup == "" { - vmCtx.Logger.V(4).Info("Skipping VM disk data backup as unchanged") + ctx.VMCtx.Logger.V(4).Info("Skipping VM disk data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ Key: constants.BackupVMDiskDataExtraConfigKey, @@ -102,12 +104,12 @@ func BackupVirtualMachine( } if len(ecToUpdate) != 0 { - vmCtx.Logger.Info("Updating VM ExtraConfig with backup data") - vmCtx.Logger.V(4).Info("", "ExtraConfig", ecToUpdate) - if _, err := vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ + ctx.VMCtx.Logger.Info("Updating VM ExtraConfig with backup data") + ctx.VMCtx.Logger.V(4).Info("", "ExtraConfig", ecToUpdate) + if _, err := ctx.VcVM.Reconfigure(ctx.VMCtx, types.VirtualMachineConfigSpec{ ExtraConfig: ecToUpdate, }); err != nil { - vmCtx.Logger.Error(err, "Failed to update VM ExtraConfig for backup") + ctx.VMCtx.Logger.Error(err, "Failed to update VM ExtraConfig for backup") return err } } @@ -194,10 +196,14 @@ func getDesiredBootstrapDataForBackup( } func getDesiredDiskDataForBackup( - ctx goctx.Context, - vcVM *object.VirtualMachine, + ctx context.BackupVirtualMachineContextA2, ecMap map[string]string) (string, error) { - deviceList, err := vcVM.Device(ctx) + // Return an empty string to skip backup if no disk uuid to PVC is specified. + if len(ctx.DiskUUIDToPVC) == 0 { + return "", nil + } + + deviceList, err := ctx.VcVM.Device(ctx.VMCtx) if err != nil { return "", err } @@ -205,16 +211,14 @@ func getDesiredDiskDataForBackup( var diskData []VMDiskData for _, device := range deviceList.SelectByType((*types.VirtualDisk)(nil)) { if disk, ok := device.(*types.VirtualDisk); ok { - vmDiskData := VMDiskData{} - if disk.VDiskId != nil { - vmDiskData.VDiskID = disk.VDiskId.Id - } if b, ok := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo); ok { - vmDiskData.FileName = b.FileName - } - // Only add the disk data if it's not empty. - if vmDiskData != (VMDiskData{}) { - diskData = append(diskData, vmDiskData) + if pvc, ok := ctx.DiskUUIDToPVC[b.Uuid]; ok { + diskData = append(diskData, VMDiskData{ + FileName: b.FileName, + PVCName: pvc.Name, + AccessModes: pvc.Spec.AccessModes, + }) + } } } } diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go index d26190ae3..2a4cbcf98 100644 --- a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go @@ -6,6 +6,7 @@ package virtualmachine_test import ( "encoding/json" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/yaml" . "github.com/onsi/ginkgo" @@ -23,6 +24,14 @@ import ( ) func backupTests() { + const ( + // These are the default values of the vcsim VM that are used to + // construct the expected backup data in the following tests. + vcSimVMPath = "DC0_C0_RP0_VM0" + vcSimDiskUUID = "be8d2471-f32e-5c7e-a89b-22cb8e533890" + vcSimDiskFileName = "[LocalDS_0] DC0_C0_RP0_VM0/disk1.vmdk" + ) + var ( ctx *builder.TestContextForVCSim vcVM *object.VirtualMachine @@ -76,7 +85,13 @@ func backupTests() { It("Should backup VM kube data YAML with the latest spec", func() { vmCtx.VM.ObjectMeta.Generation = 2 - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + backupVMCtx := context.BackupVirtualMachineContextA2{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) vmCopy := vmCtx.VM.DeepCopy() vmCopy.Status = vmopv1.VirtualMachineStatus{} @@ -112,7 +127,13 @@ func backupTests() { It("Should skip backing up VM kube data", func() { // Update the VM to verify its kube data is not backed up in ExtraConfig. vmCtx.VM.Labels = map[string]string{"foo": "bar"} - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + backupVMCtx := context.BackupVirtualMachineContextA2{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, kubeDataBackup) }) }) @@ -122,7 +143,13 @@ func backupTests() { It("Should back up bootstrap data as JSON in ExtraConfig", func() { bootstrapDataRaw := map[string]string{"foo": "bar"} - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, bootstrapDataRaw)).To(Succeed()) + backupVMCtx := context.BackupVirtualMachineContextA2{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: bootstrapDataRaw, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) bootstrapDataJSON, err := json.Marshal(bootstrapDataRaw) Expect(err).NotTo(HaveOccurred()) @@ -132,19 +159,47 @@ func backupTests() { Context("VM Disk data", func() { - It("Should backup VM disk data as JSON in ExtraConfig", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + When("VM has no disks that are attached from PVCs", func() { - // Use the default disk info from the vcSim VM for testing. - diskData := []virtualmachine.VMDiskData{ - { - VDiskID: "", - FileName: "[LocalDS_0] DC0_C0_RP0_VM0/disk1.vmdk", - }, - } - diskDataJSON, err := json.Marshal(diskData) - Expect(err).NotTo(HaveOccurred()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, string(diskDataJSON)) + It("Should skip backing up VM disk data", func() { + backupVMCtx := context.BackupVirtualMachineContextA2{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, "") + }) + }) + + When("VM has disks that are attached from PVCs", func() { + + It("Should backup VM disk data as JSON in ExtraConfig", func() { + dummyPVC := builder.DummyPersistentVolumeClaim() + diskUUIDToPVC := map[string]corev1.PersistentVolumeClaim{ + vcSimDiskUUID: *dummyPVC, + } + + backupVMCtx := context.BackupVirtualMachineContextA2{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: diskUUIDToPVC, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) + + diskData := []virtualmachine.VMDiskData{ + { + FileName: vcSimDiskFileName, + PVCName: dummyPVC.Name, + AccessModes: dummyPVC.Spec.AccessModes, + }, + } + diskDataJSON, err := json.Marshal(diskData) + Expect(err).NotTo(HaveOccurred()) + verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, string(diskDataJSON)) + }) }) }) @@ -171,8 +226,14 @@ func backupTests() { } }) - It("Should skip backing up the cloud-init instance ID", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + It("Should not change the cloud-init instance ID in VM's ExtraConfig", func() { + backupVMCtx := context.BackupVirtualMachineContextA2{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "ec-instance-id") }) }) @@ -187,7 +248,13 @@ func backupTests() { }) It("Should backup the cloud-init instance ID from annotations", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) + backupVMCtx := context.BackupVirtualMachineContextA2{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "annotation-instance-id") }) }) @@ -199,9 +266,14 @@ func backupTests() { vmCtx.VM.UID = "vm-uid" }) - It("Should backup the cloud-init instance ID from annotations", func() { - Expect(virtualmachine.BackupVirtualMachine(vmCtx, vcVM, nil)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "vm-uid") + It("Should backup the cloud-init instance ID from VM K8s resource UID", func() { + backupVMCtx := context.BackupVirtualMachineContextA2{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: nil, + DiskUUIDToPVC: nil, + } + Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) }) }) }) @@ -220,6 +292,12 @@ func verifyBackupDataInExtraConfig( Expect(objVM.Properties(ctx, objVM.Reference(), []string{"config.extraConfig"}, &moVM)).To(Succeed()) ecMap := util.ExtraConfigToMap(moVM.Config.ExtraConfig) + // Verify the expected key doesn't exist in ExtraConfig if the expected value is empty. + if expectedValDecoded == "" { + Expect(ecMap).NotTo(HaveKey(expectedKey)) + return + } + // Verify the expected key exists in ExtraConfig and the decoded values match. Expect(ecMap).To(HaveKey(expectedKey)) ecValRaw := ecMap[expectedKey] diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go index 97cf1e489..c3046d060 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go @@ -364,9 +364,23 @@ func (vs *vSphereVMProvider) updateVirtualMachine( // TODO: Support backing up vAppConfig bootstrap data. data, _, _, err := GetVirtualMachineBootstrap(vmCtx, vs.k8sClient) if err != nil { + vmCtx.Logger.Error(err, "Failed to get VM's bootstrap data for backup") return err } - if err := virtualmachine.BackupVirtualMachine(vmCtx, vcVM, data); err != nil { + diskUUIDToPVC, err := GetAttachedDiskUUIDToPVC(vmCtx, vs.k8sClient) + if err != nil { + vmCtx.Logger.Error(err, "Failed to get VM's attached PVCs for backup") + return err + } + + backupVMCtx := context.BackupVirtualMachineContextA2{ + VMCtx: vmCtx, + VcVM: vcVM, + BootstrapData: data, + DiskUUIDToPVC: diskUUIDToPVC, + } + if err := virtualmachine.BackupVirtualMachine(backupVMCtx); err != nil { + vmCtx.Logger.Error(err, "Failed to backup VM") return err } } diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go index 5506214e8..c659c6072 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go @@ -333,3 +333,43 @@ func HardwareVersionForPVCandPCIDevices(imageHWVersion int32, configSpec *types. return configSpecHWVersion } + +// GetAttachedDiskUUIDToPVC returns a map of disk UUID to PVC object for all +// attached disks by checking the VM's spec and status of volumes. +func GetAttachedDiskUUIDToPVC( + vmCtx context.VirtualMachineContextA2, + k8sClient ctrlclient.Client) (map[string]corev1.PersistentVolumeClaim, error) { + if !HasPVC(vmCtx.VM.Spec) { + return nil, nil + } + + vmVolNameToPVCName := map[string]string{} + for _, vol := range vmCtx.VM.Spec.Volumes { + if pvc := vol.PersistentVolumeClaim; pvc != nil { + vmVolNameToPVCName[vol.Name] = pvc.ClaimName + } + } + + diskUUIDToPVC := map[string]corev1.PersistentVolumeClaim{} + for _, vol := range vmCtx.VM.Status.Volumes { + if !vol.Attached || vol.DiskUUID == "" { + continue + } + + pvcName := vmVolNameToPVCName[vol.Name] + // This could happen if the volume was just removed from VM spec but not reconciled yet. + if pvcName == "" { + continue + } + + pvcObj := corev1.PersistentVolumeClaim{} + objKey := ctrlclient.ObjectKey{Name: pvcName, Namespace: vmCtx.VM.Namespace} + if err := k8sClient.Get(vmCtx, objKey, &pvcObj); err != nil { + return nil, err + } + + diskUUIDToPVC[vol.DiskUUID] = pvcObj + } + + return diskUUIDToPVC, nil +} diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go index cf7fe642d..de86efb6d 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils_test.go @@ -626,4 +626,73 @@ func vmUtilTests() { Expect(vsphere.HardwareVersionForPVCandPCIDevices(imageHWVersion, configSpec, true)).To(Equal(int32(16))) }) }) + + Context("GetAttachedDiskUuidToPVC", func() { + const ( + attachedDiskUUID = "dummy-uuid" + ) + var ( + attachedPVC *corev1.PersistentVolumeClaim + ) + + BeforeEach(func() { + // Create multiple PVCs to verity only the expected one is returned. + unusedPVC := builder.DummyPersistentVolumeClaim() + unusedPVC.Name = "unused-pvc" + unusedPVC.Namespace = vmCtx.VM.Namespace + unattachedPVC := builder.DummyPersistentVolumeClaim() + unattachedPVC.Name = "unattached-pvc" + unattachedPVC.Namespace = vmCtx.VM.Namespace + attachedPVC = builder.DummyPersistentVolumeClaim() + attachedPVC.Name = "attached-pvc" + attachedPVC.Namespace = vmCtx.VM.Namespace + initObjects = append(initObjects, unusedPVC, unattachedPVC, attachedPVC) + + vmCtx.VM.Spec.Volumes = []vmopv1.VirtualMachineVolume{ + { + Name: "unattached-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: unattachedPVC.Name, + }, + }, + }, + }, + { + Name: "attached-vol", + VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{ + PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: attachedPVC.Name, + }, + }, + }, + }, + } + + vmCtx.VM.Status.Volumes = []vmopv1.VirtualMachineVolumeStatus{ + { + Name: "unattached-vol", + Attached: false, + DiskUUID: "unattached-disk-uuid", + }, + { + Name: "attached-vol", + Attached: true, + DiskUUID: attachedDiskUUID, + }, + } + }) + + It("Should return a map of disk uuid to PVCs that are attached to the VM", func() { + diskUUIDToPVCMap, err := vsphere.GetAttachedDiskUUIDToPVC(vmCtx, k8sClient) + Expect(err).ToNot(HaveOccurred()) + Expect(diskUUIDToPVCMap).To(HaveLen(1)) + Expect(diskUUIDToPVCMap).To(HaveKey(attachedDiskUUID)) + pvc := diskUUIDToPVCMap[attachedDiskUUID] + Expect(pvc.Name).To(Equal(attachedPVC.Name)) + Expect(pvc.Namespace).To(Equal(attachedPVC.Namespace)) + }) + }) } From ee0250839ddba9def30af34d5bb7d7ba8dd5c4b6 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Thu, 26 Oct 2023 17:00:47 -0500 Subject: [PATCH 44/54] Remove the v1a2 Network top-level interface-related fields Network interfaces now can only be specified via the Interfaces list, instead of duplicating several fields between the Spec.Network and the Spec.Network.Interfaces. Similar to v1a1, use the mutation webhook to create a default interface when networking is not disabled. --- api/v1alpha1/virtualmachine_conversion.go | 1 - api/v1alpha2/virtualmachine_network_types.go | 175 +--------------- api/v1alpha2/zz_generated.deepcopy.go | 30 --- ...vmoperator.vmware.com_virtualmachines.yaml | 156 +------------- .../vsphere2/session/session_vm_update.go | 29 +-- .../providers/vsphere2/vmprovider_vm.go | 28 +-- .../providers/vsphere2/vmprovider_vm2_test.go | 26 ++- .../providers/vsphere2/vmprovider_vm_test.go | 7 +- test/builder/utila2.go | 19 +- .../mutation/virtualmachine_mutator.go | 79 ++++++- .../virtualmachine_mutator_unit_test.go | 92 ++++++++- .../validation/virtualmachine_validator.go | 31 --- .../virtualmachine_validator_unit_test.go | 192 +++++++----------- 13 files changed, 285 insertions(+), 580 deletions(-) diff --git a/api/v1alpha1/virtualmachine_conversion.go b/api/v1alpha1/virtualmachine_conversion.go index 6885a7ef3..ab09df392 100644 --- a/api/v1alpha1/virtualmachine_conversion.go +++ b/api/v1alpha1/virtualmachine_conversion.go @@ -470,7 +470,6 @@ func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( networkInterfaceSpec := convert_v1alpha1_NetworkInterface_To_v1alpha2_NetworkInterfaceSpec(i, networkInterface) out.Network.Interfaces = append(out.Network.Interfaces, networkInterfaceSpec) } - // TODO: out.Network.Network = ??? out.ReadinessProbe = convert_v1alpha1_Probe_To_v1alpha2_ReadinessProbeSpec(in.ReadinessProbe) out.Advanced = convert_v1alpha1_VirtualMachineAdvancedOptions_To_v1alpha2_VirtualMachineAdvancedSpec(in.AdvancedOptions) diff --git a/api/v1alpha2/virtualmachine_network_types.go b/api/v1alpha2/virtualmachine_network_types.go index 0b59a6af8..ae92e949f 100644 --- a/api/v1alpha2/virtualmachine_network_types.go +++ b/api/v1alpha2/virtualmachine_network_types.go @@ -160,18 +160,14 @@ type VirtualMachineNetworkInterfaceSpec struct { // VirtualMachineNetworkSpec defines a VM's desired network configuration. type VirtualMachineNetworkSpec struct { - // Network is the optional name of the network resource to which this - // VM is connected. - // - // Please note if the Interfaces field is non-empty then this field is - // ignored. + // HostName is the value the guest uses as its host name. + // If omitted then the name of the VM will be used. // - // If networking is not disabled, no interfaces are defined, and this value - // is omitted, then the VM will be provided a single virtual network - // interface and connected to the Namespace's default network. + // Please note this feature is available only with the following bootstrap + // providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). // // +optional - Network *common.PartialObjectRef `json:"network,omitempty"` + HostName string `json:"hostName,omitempty"` // Disabled is a flag that indicates whether or not to disable networking // for this VM. @@ -182,172 +178,15 @@ type VirtualMachineNetworkSpec struct { // +optional Disabled bool `json:"disabled,omitempty"` - // HostName is the value the guest uses as its host name. - // If omitted then the name of the VM will be used. - // - // Please note this feature is available only with the following bootstrap - // providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). - // - // +optional - HostName string `json:"hostName,omitempty"` - // Interfaces is the list of network interfaces used by this VM. // - // Please note this field is mutually exclusive with the following fields: - // DeviceName, Network, Addresses, DHCP4, DHCP6, Gateway4, - // Gateway6, MTU, Nameservers, Routes, and SearchDomains. + // If the Interfaces field is empty and the Disabled field is false, then + // a default interface with the name eth0 will be created. // // +optional // +listType=map // +listMapKey=name Interfaces []VirtualMachineNetworkInterfaceSpec `json:"interfaces,omitempty"` - - // DeviceName describes the unique name of this network interface, used to - // distinguish it from other network interfaces attached to this VM. - // - // This value is also used to rename the device inside the guest when the - // bootstrap provider is CloudInit. Please note it is up to the user to - // ensure the provided device name does not conflict with any other devices - // inside the guest, ex. dvd, cdrom, sda, etc. - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // If the Interfaces field is empty and this field is not specified, then - // the default interface's name will be eth0. - // - // +optional - // +kubebuilder:validation:Pattern=^\w\w+$ - DeviceName string `json:"deviceName,omitempty"` - - // Addresses is an optional list of IP4 or IP6 addresses to assign to the - // VM. - // - // Please note this field is only supported if the connected network - // supports manual IP allocation. - // - // Please note IP4 and IP6 addresses must include the network prefix length, - // ex. 192.168.0.10/24 or 2001:db8:101::a/64. - // - // Please note this field may not contain IP4 addresses if DHCP4 is set - // to true or IP6 addresses if DHCP6 is set to true. - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // +optional - Addresses []string `json:"addresses,omitempty"` - - // DHCP4 indicates whether or not to use DHCP for IP4 networking. - // - // Please note this field is only supported if the network connection - // supports DHCP. - // - // Please note this field is mutually exclusive with IP4 addresses in the - // Addresses field and the Gateway4 field. - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // +optional - DHCP4 bool `json:"dhcp4,omitempty"` - - // DHCP6 indicates whether or not to use DHCP for IP6 networking. - // - // Please note this field is only supported if the network connection - // supports DHCP. - // - // Please note this field is mutually exclusive with IP6 addresses in the - // Addresses field and the Gateway6 field. - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // +optional - DHCP6 bool `json:"dhcp6,omitempty"` - - // Gateway4 is the default, IP4 gateway for this VM. - // - // Please note this field is only supported if the network connection - // supports manual IP allocation. - // - // If the network connection supports manual IP allocation and the - // Addresses field includes at least one IP4 address, then this field - // is required. - // - // Please note this field is mutually exclusive with DHCP4. - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // +optional - Gateway4 string `json:"gateway4,omitempty"` - - // Gateway6 is the primary IP6 gateway for this VM. - // - // Please note this field is only supported if the network connection - // supports manual IP allocation. - // - // If the network connection supports manual IP allocation and the - // Addresses field includes at least one IP6 address, then this field - // is required. - // - // Please note this field is mutually exclusive with DHCP6. - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // +optional - Gateway6 string `json:"gateway6,omitempty"` - - // MTU is the Maximum Transmission Unit size in bytes. - // - // Please note this feature is available only with the following bootstrap - // providers: CloudInit. - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // +optional - MTU *int64 `json:"mtu,omitempty"` - - // Nameservers is a list of IP4 and/or IP6 addresses used as DNS - // nameservers. - // - // Please note this feature is available only with the following bootstrap - // providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). - // - // Please note that Linux allows only three nameservers - // (https://linux.die.net/man/5/resolv.conf). - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // +optional - Nameservers []string `json:"nameservers,omitempty"` - - // Routes is a list of optional, static routes. - // - // Please note this feature is available only with the following bootstrap - // providers: CloudInit. - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // +optional - Routes []VirtualMachineNetworkRouteSpec `json:"routes,omitempty"` - - // SearchDomains is a list of search domains used when resolving IP - // addresses with DNS. - // - // Please note this feature is available only with the following bootstrap - // providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). - // - // Please note if the Interfaces field is non-empty then this field is - // ignored and should be specified on the elements in the Interfaces list. - // - // +optional - SearchDomains []string `json:"searchDomains,omitempty"` } // VirtualMachineNetworkDNSStatus describes the observed state of the guest's diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index 3c067a0e9..780aaccf7 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -1150,11 +1150,6 @@ func (in *VirtualMachineNetworkRouteStatus) DeepCopy() *VirtualMachineNetworkRou // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineNetworkSpec) DeepCopyInto(out *VirtualMachineNetworkSpec) { *out = *in - if in.Network != nil { - in, out := &in.Network, &out.Network - *out = new(common.PartialObjectRef) - **out = **in - } if in.Interfaces != nil { in, out := &in.Interfaces, &out.Interfaces *out = make([]VirtualMachineNetworkInterfaceSpec, len(*in)) @@ -1162,31 +1157,6 @@ func (in *VirtualMachineNetworkSpec) DeepCopyInto(out *VirtualMachineNetworkSpec (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Addresses != nil { - in, out := &in.Addresses, &out.Addresses - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.MTU != nil { - in, out := &in.MTU, &out.MTU - *out = new(int64) - **out = **in - } - if in.Nameservers != nil { - in, out := &in.Nameservers, &out.Nameservers - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Routes != nil { - in, out := &in.Routes, &out.Routes - *out = make([]VirtualMachineNetworkRouteSpec, len(*in)) - copy(*out, *in) - } - if in.SearchDomains != nil { - in, out := &in.SearchDomains, &out.SearchDomains - *out = make([]string, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineNetworkSpec. diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml index 12a4f4341..b19be2559 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml @@ -1524,79 +1524,12 @@ spec: the VM will be assigned a single, virtual network interface that is connected to the Namespace's default network." properties: - addresses: - description: "Addresses is an optional list of IP4 or IP6 addresses - to assign to the VM. \n Please note this field is only supported - if the connected network supports manual IP allocation. \n Please - note IP4 and IP6 addresses must include the network prefix length, - ex. 192.168.0.10/24 or 2001:db8:101::a/64. \n Please note this - field may not contain IP4 addresses if DHCP4 is set to true - or IP6 addresses if DHCP6 is set to true. \n Please note if - the Interfaces field is non-empty then this field is ignored - and should be specified on the elements in the Interfaces list." - items: - type: string - type: array - deviceName: - description: "DeviceName describes the unique name of this network - interface, used to distinguish it from other network interfaces - attached to this VM. \n This value is also used to rename the - device inside the guest when the bootstrap provider is CloudInit. - Please note it is up to the user to ensure the provided device - name does not conflict with any other devices inside the guest, - ex. dvd, cdrom, sda, etc. \n Please note if the Interfaces field - is non-empty then this field is ignored and should be specified - on the elements in the Interfaces list. \n If the Interfaces - field is empty and this field is not specified, then the default - interface's name will be eth0." - pattern: ^\w\w+$ - type: string - dhcp4: - description: "DHCP4 indicates whether or not to use DHCP for IP4 - networking. \n Please note this field is only supported if the - network connection supports DHCP. \n Please note this field - is mutually exclusive with IP4 addresses in the Addresses field - and the Gateway4 field. \n Please note if the Interfaces field - is non-empty then this field is ignored and should be specified - on the elements in the Interfaces list." - type: boolean - dhcp6: - description: "DHCP6 indicates whether or not to use DHCP for IP6 - networking. \n Please note this field is only supported if the - network connection supports DHCP. \n Please note this field - is mutually exclusive with IP6 addresses in the Addresses field - and the Gateway6 field. \n Please note if the Interfaces field - is non-empty then this field is ignored and should be specified - on the elements in the Interfaces list." - type: boolean disabled: description: "Disabled is a flag that indicates whether or not to disable networking for this VM. \n When set to true, the VM is not configured with a default interface nor any specified from the Interfaces field." type: boolean - gateway4: - description: "Gateway4 is the default, IP4 gateway for this VM. - \n Please note this field is only supported if the network connection - supports manual IP allocation. \n If the network connection - supports manual IP allocation and the Addresses field includes - at least one IP4 address, then this field is required. \n Please - note this field is mutually exclusive with DHCP4. \n Please - note if the Interfaces field is non-empty then this field is - ignored and should be specified on the elements in the Interfaces - list." - type: string - gateway6: - description: "Gateway6 is the primary IP6 gateway for this VM. - \n Please note this field is only supported if the network connection - supports manual IP allocation. \n If the network connection - supports manual IP allocation and the Addresses field includes - at least one IP6 address, then this field is required. \n Please - note this field is mutually exclusive with DHCP6. \n Please - note if the Interfaces field is non-empty then this field is - ignored and should be specified on the elements in the Interfaces - list." - type: string hostName: description: "HostName is the value the guest uses as its host name. If omitted then the name of the VM will be used. \n Please @@ -1605,9 +1538,9 @@ spec: type: string interfaces: description: "Interfaces is the list of network interfaces used - by this VM. \n Please note this field is mutually exclusive - with the following fields: DeviceName, Network, Addresses, DHCP4, - DHCP6, Gateway4, Gateway6, MTU, Nameservers, Routes, and SearchDomains." + by this VM. \n If the Interfaces field is empty and the Disabled + field is false, then a default interface with the name eth0 + will be created." items: description: VirtualMachineNetworkInterfaceSpec describes the desired state of a VM's network interface. @@ -1754,89 +1687,6 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map - mtu: - description: "MTU is the Maximum Transmission Unit size in bytes. - \n Please note this feature is available only with the following - bootstrap providers: CloudInit. \n Please note if the Interfaces - field is non-empty then this field is ignored and should be - specified on the elements in the Interfaces list." - format: int64 - type: integer - nameservers: - description: "Nameservers is a list of IP4 and/or IP6 addresses - used as DNS nameservers. \n Please note this feature is available - only with the following bootstrap providers: CloudInit, LinuxPrep, - and Sysprep (except for RawSysprep). \n Please note that Linux - allows only three nameservers (https://linux.die.net/man/5/resolv.conf). - \n Please note if the Interfaces field is non-empty then this - field is ignored and should be specified on the elements in - the Interfaces list." - items: - type: string - type: array - network: - description: "Network is the optional name of the network resource - to which this VM is connected. \n Please note if the Interfaces - field is non-empty then this field is ignored. \n If networking - is not disabled, no interfaces are defined, and this value is - omitted, then the VM will be provided a single virtual network - interface and connected to the Namespace's default network." - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST - resource this object represents. Servers may infer this - from the endpoint the client submits requests to. Cannot - be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name refers to a unique resource in the current - namespace. More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - name - type: object - routes: - description: "Routes is a list of optional, static routes. \n - Please note this feature is available only with the following - bootstrap providers: CloudInit. \n Please note if the Interfaces - field is non-empty then this field is ignored and should be - specified on the elements in the Interfaces list." - items: - description: VirtualMachineNetworkRouteSpec defines a static - route for a guest. - properties: - metric: - description: Metric is the weight/priority of the route. - format: int32 - type: integer - to: - description: To is an IP4 or IP6 address. - type: string - via: - description: Via is an IP4 or IP6 address. - type: string - required: - - metric - - to - - via - type: object - type: array - searchDomains: - description: "SearchDomains is a list of search domains used when - resolving IP addresses with DNS. \n Please note this feature - is available only with the following bootstrap providers: CloudInit, - LinuxPrep, and Sysprep (except for RawSysprep). \n Please note - if the Interfaces field is non-empty then this field is ignored - and should be specified on the elements in the Interfaces list." - items: - type: string - type: array type: object nextRestartTime: description: "NextRestartTime may be used to restart the VM, in accordance diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go index d82f6ce13..6a715ab92 100644 --- a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go @@ -571,38 +571,13 @@ func (s *Session) ensureNetworkInterfaces( &vimTypes.VirtualSriovEthernetCard{}, ) } + networkSpec := &vmCtx.VM.Spec.Network if networkSpec.Disabled { // No connected networking for this VM. return network2.NetworkInterfaceResults{}, nil } - interfaces := networkSpec.Interfaces - if len(interfaces) == 0 { - // VM gets one automatic NIC. Create the default interface from fields in the network spec. - defaultInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ - Name: networkSpec.DeviceName, - Addresses: networkSpec.Addresses, - DHCP4: networkSpec.DHCP4, - DHCP6: networkSpec.DHCP6, - Gateway4: networkSpec.Gateway4, - Gateway6: networkSpec.Gateway6, - MTU: networkSpec.MTU, - Nameservers: networkSpec.Nameservers, - Routes: networkSpec.Routes, - SearchDomains: networkSpec.SearchDomains, - } - - if defaultInterface.Name == "" { - defaultInterface.Name = "eth0" - } - if networkSpec.Network != nil { - defaultInterface.Network = *networkSpec.Network - } - - interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{defaultInterface} - } - clusterMoRef := s.Cluster.Reference() results, err := network2.CreateAndWaitForNetworkInterfaces( vmCtx, @@ -610,7 +585,7 @@ func (s *Session) ensureNetworkInterfaces( s.Client.VimClient(), s.Finder, &clusterMoRef, - interfaces) + networkSpec.Interfaces) if err != nil { return network2.NetworkInterfaceResults{}, err } diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go index c3046d060..ab05c5d30 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go @@ -808,39 +808,13 @@ func (vs *vSphereVMProvider) vmCreateDoNetworking( return nil } - interfaces := networkSpec.Interfaces - if len(interfaces) == 0 { - // VM gets one automatic NIC. Create the default interface from fields in the network spec. - defaultInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ - Name: networkSpec.DeviceName, - Addresses: networkSpec.Addresses, - DHCP4: networkSpec.DHCP4, - DHCP6: networkSpec.DHCP6, - Gateway4: networkSpec.Gateway4, - Gateway6: networkSpec.Gateway6, - MTU: networkSpec.MTU, - Nameservers: networkSpec.Nameservers, - Routes: networkSpec.Routes, - SearchDomains: networkSpec.SearchDomains, - } - - if defaultInterface.Name == "" { - defaultInterface.Name = "eth0" - } - if networkSpec.Network != nil { - defaultInterface.Network = *networkSpec.Network - } - - interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{defaultInterface} - } - results, err := network.CreateAndWaitForNetworkInterfaces( vmCtx, vs.k8sClient, vcClient.VimClient(), vcClient.Finder(), nil, // Don't know the CCR yet (needed to resolve backings for NSX-T) - interfaces) + networkSpec.Interfaces) if err != nil { conditions.MarkFalse(vmCtx.VM, vmopv1.VirtualMachineConditionNetworkReady, "NotReady", err.Error()) return err diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go index 5b516a6cb..28f121374 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm2_test.go @@ -87,8 +87,8 @@ func vmE2ETests() { vm.Spec.ClassName = vmClass.Name vm.Spec.ImageName = ctx.ContentLibraryImageName vm.Spec.StorageClass = ctx.StorageClassName - vm.Spec.Network.Nameservers = []string{"1.1.1.1", "8.8.8.8"} - vm.Spec.Network.SearchDomains = []string{"vmware.local"} + vm.Spec.Network.Interfaces[0].Nameservers = []string{"1.1.1.1", "8.8.8.8"} + vm.Spec.Network.Interfaces[0].SearchDomains = []string{"vmware.local"} vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{ RawCloudConfig: corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ @@ -119,8 +119,15 @@ func vmE2ETests() { BeforeEach(func() { testConfig.WithNetworkEnv = builder.NetworkEnvVDS - vm.Spec.Network.Network = &common.PartialObjectRef{ - Name: networkName, + vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: common.PartialObjectRef{ + Name: networkName, + }, + }, + }, } }) @@ -191,8 +198,15 @@ func vmE2ETests() { BeforeEach(func() { testConfig.WithNetworkEnv = builder.NetworkEnvNSXT - vm.Spec.Network.Network = &common.PartialObjectRef{ - Name: networkName, + vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: interfaceName, + Network: common.PartialObjectRef{ + Name: networkName, + }, + }, + }, } }) diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go index cfb000f8d..19a2c15a7 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go @@ -187,7 +187,12 @@ func vmTests() { } vm.Spec.Network.Disabled = false - vm.Spec.Network.Network = &common.PartialObjectRef{Name: dvpgName} + vm.Spec.Network.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Network: common.PartialObjectRef{Name: dvpgName}, + }, + } var err error vcVM, err = createOrUpdateAndGetVcVM(ctx, vm) diff --git a/test/builder/utila2.go b/test/builder/utila2.go index 141d93cf5..771e4bfe9 100644 --- a/test/builder/utila2.go +++ b/test/builder/utila2.go @@ -156,22 +156,13 @@ func DummyVirtualMachineA2() *vmopv1.VirtualMachine { }, }, }, - /* TODO: Convert this if/as needed - NetworkInterfaces: []vmopv1.VirtualMachineNetworkInterface{ - { - NetworkName: DummyNetworkName, - NetworkType: "", - }, - { - NetworkName: DummyNetworkName + "-2", - NetworkType: "", + Network: vmopv1.VirtualMachineNetworkSpec{ + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + }, }, }, - VmMetadata: &vmopv1.VirtualMachineMetadata{ - ConfigMapName: DummyMetadataCMName, - Transport: "ExtraConfig", - }, - */ }, } } diff --git a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go index 612e1a9c0..b39d3fcde 100644 --- a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go +++ b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator.go @@ -12,6 +12,8 @@ import ( "github.com/pkg/errors" admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" @@ -19,14 +21,22 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + ncpv1alpha1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" + netopv1alpha1 "github.com/vmware-tanzu/vm-operator/external/net-operator/api/v1alpha1" + + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" "github.com/vmware-tanzu/vm-operator/pkg/builder" "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" ) const ( - webHookName = "default" + webHookName = "default" + defaultInterfaceName = "eth0" + defaultNamedNetwork = "VM Network" ) // +kubebuilder:webhook:path=/default-mutate-vmoperator-vmware-com-v1alpha2-virtualmachine,mutating=true,failurePolicy=fail,groups=vmoperator.vmware.com,resources=virtualmachines,verbs=create;update,versions=v1alpha2,name=default.mutating.virtualmachine.v1alpha2.vmoperator.vmware.com,sideEffects=None,admissionReviewVersions=v1;v1beta1 @@ -100,6 +110,9 @@ func (m mutator) Mutate(ctx *context.WebhookRequestContext) admission.Response { switch ctx.Op { case admissionv1.Create: + if AddDefaultNetworkInterface(ctx, m.client, modified) { + wasMutated = true + } if mutated, err := ResolveImageName(ctx, m.client, modified); err != nil { return admission.Denied(err.Error()) } else if mutated { @@ -176,6 +189,70 @@ func SetNextRestartTime( `may only be set to "now"`) } +// AddDefaultNetworkInterface adds default network interface to a VM if the NoNetwork annotation is not set +// and no NetworkInterface is specified. +// Return true if default NetworkInterface is added, otherwise return false. +func AddDefaultNetworkInterface(ctx *context.WebhookRequestContext, client client.Client, vm *vmopv1.VirtualMachine) bool { + // Continue to support this ad-hoc v1a1 annotation. I don't think need or want to have this annotation + // in v1a2: Disabled mostly already covers it. We could map between the two for version conversion, but + // they do mean slightly different things, and kind of complicated to know what to do like if the annotation + // is removed. + if _, ok := vm.Annotations[v1alpha1.NoDefaultNicAnnotation]; ok { + return false + } + + if vm.Spec.Network.Disabled || len(vm.Spec.Network.Interfaces) != 0 { + return false + } + + kind, apiVersion, netName := "", "", "" + switch lib.GetNetworkProviderType() { + case lib.NetworkProviderTypeNSXT: + kind = "VirtualNetwork" + apiVersion = ncpv1alpha1.SchemeGroupVersion.String() + case lib.NetworkProviderTypeVDS: + kind = "Network" + apiVersion = netopv1alpha1.SchemeGroupVersion.String() + case lib.NetworkProviderTypeNamed: + netName, _ = getProviderConfigMap(ctx, client) + if netName == "" { + netName = defaultNamedNetwork + } + default: + return false + } + + vm.Spec.Network.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: defaultInterfaceName, + Network: common.PartialObjectRef{ + TypeMeta: metav1.TypeMeta{ + Kind: kind, + APIVersion: apiVersion, + }, + Name: netName, + }, + }, + } + + return true +} + +// getProviderConfigMap is used in e2e tests. +func getProviderConfigMap(ctx *context.WebhookRequestContext, c client.Client) (string, error) { + var obj corev1.ConfigMap + if err := c.Get( + ctx, + client.ObjectKey{ + Name: config.ProviderConfigMapName, + Namespace: ctx.Namespace, + }, + &obj); err != nil { + return "", err + } + return obj.Data["Network"], nil +} + // ResolveImageName mutates the vm.spec.imageName if it's not set to a vmi name // and there is a single namespace or cluster scope image with that status name. func ResolveImageName( diff --git a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_unit_test.go b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_unit_test.go index b3515fbeb..405c049bf 100644 --- a/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha2/mutation/virtualmachine_mutator_unit_test.go @@ -11,13 +11,16 @@ import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/vmware-tanzu/vm-operator/api/v1alpha1" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/config" "github.com/vmware-tanzu/vm-operator/test/builder" "github.com/vmware-tanzu/vm-operator/webhooks/virtualmachine/v1alpha2/mutation" ) @@ -52,6 +55,7 @@ func unitTestsMutating() { }) AfterEach(func() { + Expect(os.Unsetenv(lib.NetworkProviderType)).To(Succeed()) ctx = nil }) @@ -66,6 +70,92 @@ func unitTestsMutating() { }) }) + Describe("AddDefaultNetworkInterface", func() { + + Context("When VM NetworkInterface is empty", func() { + BeforeEach(func() { + ctx.vm.Spec.Network.Interfaces = []vmopv1.VirtualMachineNetworkInterfaceSpec{} + }) + + When("VDS network", func() { + BeforeEach(func() { + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeVDS)).Should(Succeed()) + }) + + It("Should add default network interface with type vsphere-distributed", func() { + Expect(mutation.AddDefaultNetworkInterface(&ctx.WebhookRequestContext, ctx.Client, ctx.vm)).To(BeTrue()) + Expect(ctx.vm.Spec.Network.Interfaces).Should(HaveLen(1)) + Expect(ctx.vm.Spec.Network.Interfaces[0].Name).Should(Equal("eth0")) + Expect(ctx.vm.Spec.Network.Interfaces[0].Network.Kind).Should(Equal("Network")) + }) + }) + + When("NSX-T network", func() { + BeforeEach(func() { + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeNSXT)).Should(Succeed()) + }) + + It("Should add default network interface with type NSX-T", func() { + Expect(mutation.AddDefaultNetworkInterface(&ctx.WebhookRequestContext, ctx.Client, ctx.vm)).To(BeTrue()) + Expect(ctx.vm.Spec.Network.Interfaces).Should(HaveLen(1)) + Expect(ctx.vm.Spec.Network.Interfaces[0].Name).Should(Equal("eth0")) + Expect(ctx.vm.Spec.Network.Interfaces[0].Network.Kind).Should(Equal("VirtualNetwork")) + }) + }) + + When("Named network", func() { + const networkName = "VM Network" + + BeforeEach(func() { + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeNamed)).To(Succeed()) + }) + + It("Should add default network interface with name set in the configMap Network", func() { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "vmware-system-vmop", + Name: config.ProviderConfigMapName, + }, + Data: map[string]string{"Network": networkName}, + } + Expect(ctx.Client.Create(ctx, configMap)).To(Succeed()) + + Expect(mutation.AddDefaultNetworkInterface(&ctx.WebhookRequestContext, ctx.Client, ctx.vm)).To(BeTrue()) + Expect(ctx.vm.Spec.Network.Interfaces).To(HaveLen(1)) + Expect(ctx.vm.Spec.Network.Interfaces[0].Network.Kind).To(BeEmpty()) + Expect(ctx.vm.Spec.Network.Interfaces[0].Network.Name).To(Equal(networkName)) + }) + }) + + When("NoNetwork annotation is set", func() { + BeforeEach(func() { + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeVDS)).Should(Succeed()) + }) + + It("Should not add default network interface", func() { + ctx.vm.Annotations[v1alpha1.NoDefaultNicAnnotation] = "true" + oldVM := ctx.vm.DeepCopy() + Expect(mutation.AddDefaultNetworkInterface(&ctx.WebhookRequestContext, ctx.Client, ctx.vm)).To(BeFalse()) + Expect(ctx.vm.Spec.Network.Interfaces).To(Equal(oldVM.Spec.Network.Interfaces)) + }) + }) + }) + + Context("VM NetworkInterface is not empty", func() { + BeforeEach(func() { + Expect(os.Setenv(lib.NetworkProviderType, lib.NetworkProviderTypeVDS)).Should(Succeed()) + }) + + It("Should not add default network interface", func() { + Expect(ctx.vm.Spec.Network.Interfaces).ToNot(BeEmpty()) + + oldVM := ctx.vm.DeepCopy() + Expect(mutation.AddDefaultNetworkInterface(&ctx.WebhookRequestContext, ctx.Client, ctx.vm)).To(BeFalse()) + Expect(ctx.vm.Spec.Network.Interfaces).To(Equal(oldVM.Spec.Network.Interfaces)) + }) + }) + }) + Describe("SetNextRestartTime", func() { var ( @@ -227,7 +317,7 @@ func unitTestsMutating() { }) }) - Describe(("ResolveImageName"), func() { + Describe("ResolveImageName", func() { const ( dupImageStatusName = "dup-status-name" uniqueImageStatusName = "unique-status-name" diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index 59b3a9925..33a44fbce 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -314,40 +314,9 @@ func (v validator) validateNetwork(ctx *context.WebhookRequestContext, vm *vmopv networkSpec := &vm.Spec.Network networkPath := field.NewPath("spec", "network") - defaultNetworkInterface := vmopv1.VirtualMachineNetworkInterfaceSpec{ - Name: networkSpec.DeviceName, - Addresses: networkSpec.Addresses, - DHCP4: networkSpec.DHCP4, - DHCP6: networkSpec.DHCP6, - Gateway4: networkSpec.Gateway4, - Gateway6: networkSpec.Gateway6, - MTU: networkSpec.MTU, - Nameservers: networkSpec.Nameservers, - Routes: networkSpec.Routes, - SearchDomains: networkSpec.SearchDomains, - } - if networkSpec.Network != nil { - defaultNetworkInterface.Network = *networkSpec.Network - } - hasDefaultInterface := !equality.Semantic.DeepEqual(defaultNetworkInterface, vmopv1.VirtualMachineNetworkInterfaceSpec{}) - - if hasDefaultInterface { - if defaultNetworkInterface.Name == "" { - defaultNetworkInterface.Name = "eth0" - } - allErrs = append(allErrs, v.validateNetworkInterfaceSpec(networkPath, defaultNetworkInterface, vm.Name)...) - } - if len(networkSpec.Interfaces) > 0 { p := networkPath.Child("interfaces") - if hasDefaultInterface { - // TODO: Better phrasing of this error message? - allErrs = append(allErrs, field.Invalid(p, nil, - "interfaces are mutually exclusive with deviceName,network,addresses,dhcp4,dhcp6,gateway4,"+ - "gateway6,mtu,nameservers,routes,searchDomains fields")) - } - for i, interfaceSpec := range networkSpec.Interfaces { allErrs = append(allErrs, v.validateNetworkInterfaceSpec(p.Index(i), interfaceSpec, vm.Name)...) } diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go index d6c7d4a43..aa9cee264 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go @@ -452,33 +452,37 @@ func unitTestsValidateCreate() { testParams{ setup: func(ctx *unitValidatingWebhookContext) { ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ - HostName: "my-vm", - DeviceName: "eth0", - Addresses: []string{ - "192.168.1.100/24", - "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", - }, - DHCP4: false, - DHCP6: false, - Gateway4: "192.168.1.1", - Gateway6: "2605:a601:a0ba:720:2ce6::1", - MTU: pointer.Int64(9000), - Nameservers: []string{ - "8.8.8.8", - "2001:4860:4860::8888", - }, - Routes: []vmopv1.VirtualMachineNetworkRouteSpec{ - { - To: "10.100.10.1/24", - Via: "10.10.1.1", - Metric: 42, - }, + HostName: "my-vm", + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ { - To: "fbd6:93e7:bc11:18b2:514f:2b1d:637a:f695/48", - Via: "ef71:6ce2:3b91:8349:b2b2:f76c:86ae:915b", + Name: "eth0", + Addresses: []string{ + "192.168.1.100/24", + "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", + }, + DHCP4: false, + DHCP6: false, + Gateway4: "192.168.1.1", + Gateway6: "2605:a601:a0ba:720:2ce6::1", + MTU: pointer.Int64(9000), + Nameservers: []string{ + "8.8.8.8", + "2001:4860:4860::8888", + }, + Routes: []vmopv1.VirtualMachineNetworkRouteSpec{ + { + To: "10.100.10.1/24", + Via: "10.10.1.1", + Metric: 42, + }, + { + To: "fbd6:93e7:bc11:18b2:514f:2b1d:637a:f695/48", + Via: "ef71:6ce2:3b91:8349:b2b2:f76c:86ae:915b", + }, + }, + SearchDomains: []string{"dev.local"}, }, }, - SearchDomains: []string{"dev.local"}, } }, expectAllowed: true, @@ -489,10 +493,14 @@ func unitTestsValidateCreate() { testParams{ setup: func(ctx *unitValidatingWebhookContext) { ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ - HostName: "my-vm", - DeviceName: "eth0", - DHCP4: true, - DHCP6: true, + HostName: "my-vm", + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + DHCP4: true, + DHCP6: true, + }, + }, } }, expectAllowed: true, @@ -503,23 +511,27 @@ func unitTestsValidateCreate() { testParams{ setup: func(ctx *unitValidatingWebhookContext) { ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ - HostName: "my-vm", - DeviceName: "eth0", - Addresses: []string{ - "192.168.1.100/24", - "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", + HostName: "my-vm", + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Addresses: []string{ + "192.168.1.100/24", + "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", + }, + DHCP4: true, + DHCP6: true, + Gateway4: "192.168.1.1", + Gateway6: "2605:a601:a0ba:720:2ce6::1", + }, }, - DHCP4: true, - DHCP6: true, - Gateway4: "192.168.1.1", - Gateway6: "2605:a601:a0ba:720:2ce6::1", } }, validate: doValidateWithMsg( - `spec.network.dhcp4: Invalid value: "192.168.1.100/24": dhcp4 cannot be used with IPv4 addresses in addresses field`, - `spec.network.gateway4: Invalid value: "192.168.1.1": gateway4 is mutually exclusive with dhcp4`, - `spec.network.dhcp6: Invalid value: "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48": dhcp6 cannot be used with IPv6 addresses in addresses field`, - `spec.network.gateway6: Invalid value: "2605:a601:a0ba:720:2ce6::1": gateway6 is mutually exclusive with dhcp6`, + `spec.network.interfaces[0].dhcp4: Invalid value: "192.168.1.100/24": dhcp4 cannot be used with IPv4 addresses in addresses field`, + `spec.network.interfaces[0].gateway4: Invalid value: "192.168.1.1": gateway4 is mutually exclusive with dhcp4`, + `spec.network.interfaces[0].dhcp6: Invalid value: "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48": dhcp6 cannot be used with IPv6 addresses in addresses field`, + `spec.network.interfaces[0].gateway6: Invalid value: "2605:a601:a0ba:720:2ce6::1": gateway6 is mutually exclusive with dhcp6`, ), }, ), @@ -527,7 +539,7 @@ func unitTestsValidateCreate() { Entry("validate addresses", testParams{ setup: func(ctx *unitValidatingWebhookContext) { - ctx.vm.Spec.Network.Addresses = []string{ + ctx.vm.Spec.Network.Interfaces[0].Addresses = []string{ "1.1.", "1.1.1.1", "not-an-ip", @@ -535,10 +547,10 @@ func unitTestsValidateCreate() { } }, validate: doValidateWithMsg( - `spec.network.addresses[0]: Invalid value: "1.1.": invalid CIDR address: 1.1.`, - `spec.network.addresses[1]: Invalid value: "1.1.1.1": invalid CIDR address: 1.1.1.1`, - `spec.network.addresses[2]: Invalid value: "not-an-ip": invalid CIDR address: not-an-ip`, - `spec.network.addresses[3]: Invalid value: "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072": invalid CIDR address: 7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072`, + `spec.network.interfaces[0].addresses[0]: Invalid value: "1.1.": invalid CIDR address: 1.1.`, + `spec.network.interfaces[0].addresses[1]: Invalid value: "1.1.1.1": invalid CIDR address: 1.1.1.1`, + `spec.network.interfaces[0].addresses[2]: Invalid value: "not-an-ip": invalid CIDR address: not-an-ip`, + `spec.network.interfaces[0].addresses[3]: Invalid value: "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072": invalid CIDR address: 7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072`, ), }, ), @@ -546,11 +558,11 @@ func unitTestsValidateCreate() { Entry("validate gateway4", testParams{ setup: func(ctx *unitValidatingWebhookContext) { - ctx.vm.Spec.Network.Gateway4 = "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072" + ctx.vm.Spec.Network.Interfaces[0].Gateway4 = "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072" }, validate: doValidateWithMsg( - `spec.network.gateway4: Invalid value: "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072": gateway4 must have an IPv4 address in the addresses field`, - `spec.network.gateway4: Invalid value: "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072": must be a valid IPv4 address`, + `spec.network.interfaces[0].gateway4: Invalid value: "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072": gateway4 must have an IPv4 address in the addresses field`, + `spec.network.interfaces[0].gateway4: Invalid value: "7936:39e1:d51b:39d2:05f8:1fb2:35cc:1072": must be a valid IPv4 address`, ), }, ), @@ -558,11 +570,11 @@ func unitTestsValidateCreate() { Entry("validate gateway6", testParams{ setup: func(ctx *unitValidatingWebhookContext) { - ctx.vm.Spec.Network.Gateway6 = "192.168.1.1" + ctx.vm.Spec.Network.Interfaces[0].Gateway6 = "192.168.1.1" }, validate: doValidateWithMsg( - `spec.network.gateway6: Invalid value: "192.168.1.1": gateway6 must have an IPv6 address in the addresses field`, - `spec.network.gateway6: Invalid value: "192.168.1.1": must be a valid IPv6 address`, + `spec.network.interfaces[0].gateway6: Invalid value: "192.168.1.1": gateway6 must have an IPv6 address in the addresses field`, + `spec.network.interfaces[0].gateway6: Invalid value: "192.168.1.1": must be a valid IPv6 address`, ), }, ), @@ -570,14 +582,14 @@ func unitTestsValidateCreate() { Entry("validate nameservers", testParams{ setup: func(ctx *unitValidatingWebhookContext) { - ctx.vm.Spec.Network.Nameservers = []string{ + ctx.vm.Spec.Network.Interfaces[0].Nameservers = []string{ "not-an-ip", "192.168.1.1/24", } }, validate: doValidateWithMsg( - `spec.network.nameservers[0]: Invalid value: "not-an-ip": must be an IPv4 or IPv6 address`, - `spec.network.nameservers[1]: Invalid value: "192.168.1.1/24": must be an IPv4 or IPv6 address`, + `spec.network.interfaces[0].nameservers[0]: Invalid value: "not-an-ip": must be an IPv4 or IPv6 address`, + `spec.network.interfaces[0].nameservers[1]: Invalid value: "192.168.1.1/24": must be an IPv4 or IPv6 address`, ), }, ), @@ -585,7 +597,7 @@ func unitTestsValidateCreate() { Entry("validate routes", testParams{ setup: func(ctx *unitValidatingWebhookContext) { - ctx.vm.Spec.Network.Routes = []vmopv1.VirtualMachineNetworkRouteSpec{ + ctx.vm.Spec.Network.Interfaces[0].Routes = []vmopv1.VirtualMachineNetworkRouteSpec{ { To: "10.100.10.1", Via: "192.168.1", @@ -601,70 +613,10 @@ func unitTestsValidateCreate() { } }, validate: doValidateWithMsg( - `spec.network.routes[0].to: Invalid value: "10.100.10.1": invalid CIDR address: 10.100.10.1`, - `spec.network.routes[0].via: Invalid value: "192.168.1": must be an IPv4 or IPv6 address`, - `spec.network.routes[1].via: Invalid value: "2463:foobar": must be an IPv4 or IPv6 address`, - `spec.network.routes[2]: Invalid value: "": cannot mix IP address families`, - ), - }, - ), - - Entry("validate interfaces", - testParams{ - setup: func(ctx *unitValidatingWebhookContext) { - ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ - HostName: "my-vm", - Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ - { - Name: "eth1", - Addresses: []string{ - "192.168.1.100/24", - "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", - }, - Gateway4: "192.168.1.1", - Gateway6: "2605:a601:a0ba:720:2ce6::1", - MTU: pointer.Int64(9000), - Nameservers: []string{ - "8.8.8.8", - "2001:4860:4860::8888", - }, - Routes: []vmopv1.VirtualMachineNetworkRouteSpec{ - { - To: "10.100.10.1/24", - Via: "10.10.1.1", - Metric: 42, - }, - { - To: "fbd6:93e7:bc11:18b2:514f:2b1d:637a:f695/48", - Via: "ef71:6ce2:3b91:8349:b2b2:f76c:86ae:915b", - }, - }, - SearchDomains: []string{"dev.local"}, - }, - }, - } - }, - expectAllowed: true, - }, - ), - - Entry("disallow interfaces with default interface", - testParams{ - setup: func(ctx *unitValidatingWebhookContext) { - ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ - HostName: "my-vm", - Addresses: []string{"192.168.1.10/24"}, - Gateway4: "192.168.1.1", - Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ - { - Name: "eth1", - DHCP4: true, - }, - }, - } - }, - validate: doValidateWithMsg( - `spec.network.interfaces: Invalid value: "null": interfaces are mutually exclusive with deviceName,network,addresses,dhcp4,dhcp6,gateway4,gateway6,mtu,nameservers,routes,searchDomains fields`, + `spec.network.interfaces[0].routes[0].to: Invalid value: "10.100.10.1": invalid CIDR address: 10.100.10.1`, + `spec.network.interfaces[0].routes[0].via: Invalid value: "192.168.1": must be an IPv4 or IPv6 address`, + `spec.network.interfaces[0].routes[1].via: Invalid value: "2463:foobar": must be an IPv4 or IPv6 address`, + `spec.network.interfaces[0].routes[2]: Invalid value: "": cannot mix IP address families`, ), }, ), From f65ef245599f1b71fc902edbc5f15b5118bafd28 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Fri, 3 Nov 2023 14:52:35 -0500 Subject: [PATCH 45/54] Version conversion support for v1a2 BootDiskCapacity In v1a1 we added the VsphereVolume to allow the resize of the OVF disks. It was something done quickly, and really never ended up being used, and the interface isn't very user friendly for general purpose for "VM Service" VMs. In v1a2 we just have a single field for resizing the boot disk. As best as we're able to propagate the boot disk field across versions. --- api/v1alpha1/virtualmachine_conversion.go | 69 ++++++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/api/v1alpha1/virtualmachine_conversion.go b/api/v1alpha1/virtualmachine_conversion.go index ab09df392..dc1253e09 100644 --- a/api/v1alpha1/virtualmachine_conversion.go +++ b/api/v1alpha1/virtualmachine_conversion.go @@ -10,6 +10,7 @@ import ( corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiconversion "k8s.io/apimachinery/pkg/conversion" "k8s.io/utils/pointer" @@ -18,6 +19,11 @@ import ( "github.com/vmware-tanzu/vm-operator/api/v1alpha2" ) +const ( + // Well known device key used for the first disk. + bootDiskDeviceKey = 2000 +) + func Convert_v1alpha1_VirtualMachineVolume_To_v1alpha2_VirtualMachineVolume( in *VirtualMachineVolume, out *v1alpha2.VirtualMachineVolume, s apiconversion.Scope) error { @@ -36,7 +42,7 @@ func Convert_v1alpha1_VirtualMachineVolume_To_v1alpha2_VirtualMachineVolume( } } - // TODO: in.VsphereVolume + // NOTE: in.VsphereVolume is dropped in v1a2. See filter_out_VirtualMachineVolumes_VsphereVolumes(). return autoConvert_v1alpha1_VirtualMachineVolume_To_v1alpha2_VirtualMachineVolume(in, out, s) } @@ -328,8 +334,6 @@ func convert_v1alpha1_VirtualMachineAdvancedOptions_To_v1alpha2_VirtualMachineAd out := v1alpha2.VirtualMachineAdvancedSpec{} if in != nil { - // out.BootDiskCapacity = - if opts := in.DefaultVolumeProvisioningOptions; opts != nil { if opts.ThinProvisioned != nil { if *opts.ThinProvisioned { @@ -350,6 +354,26 @@ func convert_v1alpha1_VirtualMachineAdvancedOptions_To_v1alpha2_VirtualMachineAd return out } +func convert_v1alpha1_VsphereVolumes_To_v1alpah2_BootDiskCapacity(volumes []VirtualMachineVolume) resource.Quantity { + // The v1a1 VsphereVolume was never a great API as you had to know the DeviceKey upfront; at the time our + // API was private - only used by CAPW - and predates the "VM Service" VMs; In v1a2, we only support resizing + // the boot disk via an explicit field. As good as we can here, map v1a1 volume into the v1a2 specific field. + + for i := range volumes { + vsVol := volumes[i].VsphereVolume + + if vsVol != nil && vsVol.DeviceKey != nil && *vsVol.DeviceKey == bootDiskDeviceKey { + // This VsphereVolume has the well-known boot disk device key. Return that capacity if set. + if capacity := vsVol.Capacity.StorageEphemeral(); capacity != nil { + return *capacity + } + break + } + } + + return resource.Quantity{} +} + func convert_v1alpha2_VirtualMachineAdvancedSpec_To_v1alpha1_VirtualMachineAdvancedOptions( in v1alpha2.VirtualMachineAdvancedSpec) *VirtualMachineAdvancedOptions { @@ -381,6 +405,24 @@ func convert_v1alpha2_VirtualMachineAdvancedSpec_To_v1alpha1_VirtualMachineAdvan return out } +func convert_v1alpha2_BootDiskCapacity_To_v1alpha1_VirtualMachineVolume(capacity resource.Quantity) *VirtualMachineVolume { + if capacity.IsZero() { + return nil + } + + const name = "vmoperator-vm-boot-disk" + + return &VirtualMachineVolume{ + Name: name, + VsphereVolume: &VsphereVolumeSource{ + Capacity: corev1.ResourceList{ + corev1.ResourceEphemeralStorage: capacity, + }, + DeviceKey: pointer.Int(bootDiskDeviceKey), + }, + } +} + func convert_v1alpha1_Network_To_v1alpha2_NetworkStatus( vmIP string, in []NetworkInterfaceStatus) *v1alpha2.VirtualMachineNetworkStatus { @@ -450,6 +492,21 @@ func convert_v1alpha2_NetworkStatus_To_v1alpha1_Network( return vmIP, out } +// In v1a2 we've dropped the v1a1 VsphereVolumes, and in its place we have a single field for the boot +// disk size. The Convert_v1alpha1_VirtualMachineVolume_To_v1alpha2_VirtualMachineVolume() stub does not +// allow us to not return something so filter those volumes - without a PersistentVolumeClaim set - here. +func filter_out_VirtualMachineVolumes_VsphereVolumes(volumes []v1alpha2.VirtualMachineVolume) []v1alpha2.VirtualMachineVolume { + out := make([]v1alpha2.VirtualMachineVolume, 0, len(volumes)) + + for _, v := range volumes { + if v.PersistentVolumeClaim != nil { + out = append(out, v) + } + } + + return out +} + func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( in *VirtualMachineSpec, out *v1alpha2.VirtualMachineSpec, s apiconversion.Scope) error { @@ -465,6 +522,7 @@ func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( out.NextRestartTime = in.NextRestartTime out.RestartMode = convert_v1alpha1_VirtualMachinePowerOpMode_To_v1alpha2_VirtualMachinePowerOpMode(in.RestartMode) out.Bootstrap = convert_v1alpha1_VmMetadata_To_v1alpha2_BootstrapSpec(in.VmMetadata) + out.Volumes = filter_out_VirtualMachineVolumes_VsphereVolumes(out.Volumes) for i, networkInterface := range in.NetworkInterfaces { networkInterfaceSpec := convert_v1alpha1_NetworkInterface_To_v1alpha2_NetworkInterfaceSpec(i, networkInterface) @@ -473,6 +531,7 @@ func Convert_v1alpha1_VirtualMachineSpec_To_v1alpha2_VirtualMachineSpec( out.ReadinessProbe = convert_v1alpha1_Probe_To_v1alpha2_ReadinessProbeSpec(in.ReadinessProbe) out.Advanced = convert_v1alpha1_VirtualMachineAdvancedOptions_To_v1alpha2_VirtualMachineAdvancedSpec(in.AdvancedOptions) + out.Advanced.BootDiskCapacity = convert_v1alpha1_VsphereVolumes_To_v1alpah2_BootDiskCapacity(in.Volumes) out.Reserved.ResourcePolicyName = in.ResourcePolicyName // Deprecated: @@ -504,6 +563,10 @@ func Convert_v1alpha2_VirtualMachineSpec_To_v1alpha1_VirtualMachineSpec( out.AdvancedOptions = convert_v1alpha2_VirtualMachineAdvancedSpec_To_v1alpha1_VirtualMachineAdvancedOptions(in.Advanced) out.ResourcePolicyName = in.Reserved.ResourcePolicyName + if bootDiskVol := convert_v1alpha2_BootDiskCapacity_To_v1alpha1_VirtualMachineVolume(in.Advanced.BootDiskCapacity); bootDiskVol != nil { + out.Volumes = append(out.Volumes, *bootDiskVol) + } + // TODO = in.ReadinessGates // Deprecated: From 5619de10c3d27a6c41c13f524a15ca364d7c93b1 Mon Sep 17 00:00:00 2001 From: Yiyi Zhou <91219164+zyiyi11@users.noreply.github.com> Date: Tue, 7 Nov 2023 10:45:31 -0800 Subject: [PATCH 46/54] Add validations for Networkspec with its supported Bootstrap and unit tests (#262) --- .../vsphere2/vmlifecycle/bootstrap.go | 8 +- .../providers/vsphere2/vmprovider_vm_utils.go | 4 +- .../validation/virtualmachine_validator.go | 58 ++- .../virtualmachine_validator_unit_test.go | 384 ++++++++++++++---- 4 files changed, 377 insertions(+), 77 deletions(-) diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go index 8e9f8e289..0ba50fa8d 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/bootstrap.go @@ -55,7 +55,7 @@ func DoBootstrap( bootstrap := &vmCtx.VM.Spec.Bootstrap cloudInit := bootstrap.CloudInit linuxPrep := bootstrap.LinuxPrep - sysPrep := bootstrap.Sysprep + sysprep := bootstrap.Sysprep vAppConfig := bootstrap.VAppConfig bootstrapArgs, err := getBootstrapArgs(vmCtx, k8sClient, cloudInit != nil, networkResults, bootstrapData) @@ -63,7 +63,7 @@ func DoBootstrap( return err } - if sysPrep != nil || vAppConfig != nil { + if sysprep != nil || vAppConfig != nil { bootstrapArgs.TemplateRenderFn = GetTemplateRenderFunc(vmCtx, bootstrapArgs) } @@ -75,8 +75,8 @@ func DoBootstrap( configSpec, customSpec, err = BootStrapCloudInit(vmCtx, config, cloudInit, bootstrapArgs) case linuxPrep != nil: configSpec, customSpec, err = BootStrapLinuxPrep(vmCtx, config, linuxPrep, vAppConfig, bootstrapArgs) - case sysPrep != nil: - configSpec, customSpec, err = BootstrapSysPrep(vmCtx, config, sysPrep, vAppConfig, bootstrapArgs) + case sysprep != nil: + configSpec, customSpec, err = BootstrapSysPrep(vmCtx, config, sysprep, vAppConfig, bootstrapArgs) case vAppConfig != nil: configSpec, customSpec, err = BootstrapVAppConfig(vmCtx, config, vAppConfig, bootstrapArgs) default: diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go index c659c6072..df166f2c2 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_utils.go @@ -153,8 +153,8 @@ func GetVirtualMachineBootstrap( if cloudInit := bootstrapSpec.CloudInit; cloudInit != nil { secretName = cloudInit.RawCloudConfig.Name - } else if sysPrep := bootstrapSpec.Sysprep; sysPrep != nil { - secretName = sysPrep.RawSysprep.Name + } else if sysprep := bootstrapSpec.Sysprep; sysprep != nil { + secretName = sysprep.RawSysprep.Name } if secretName != "" { diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index 33a44fbce..bd610f3e6 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -233,7 +233,7 @@ func (v validator) validateBootstrap( "sysprep and rawSysprep are mutually exclusive")) } } else { - allErrs = append(allErrs, field.Invalid(p, "sysprep", fmt.Sprintf(featureNotEnabled, "sysprep"))) + allErrs = append(allErrs, field.Invalid(p, "Sysprep", fmt.Sprintf(featureNotEnabled, "Sysprep"))) } } @@ -319,6 +319,7 @@ func (v validator) validateNetwork(ctx *context.WebhookRequestContext, vm *vmopv for i, interfaceSpec := range networkSpec.Interfaces { allErrs = append(allErrs, v.validateNetworkInterfaceSpec(p.Index(i), interfaceSpec, vm.Name)...) + allErrs = append(allErrs, v.validateNetworkSpecWithBootstrap(p.Index(i), interfaceSpec, vm)...) } } @@ -443,6 +444,61 @@ func (v validator) validateNetworkInterfaceSpec( return allErrs } +// mtu and routes is available only with CloudInit bootstrap providers. +// nameservers and searchDomains is available only with the following bootstrap +// providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). +func (v validator) validateNetworkSpecWithBootstrap( + interfacePath *field.Path, + interfaceSpec vmopv1.VirtualMachineNetworkInterfaceSpec, + vm *vmopv1.VirtualMachine) field.ErrorList { + + var allErrs field.ErrorList + cloudInit := vm.Spec.Bootstrap.CloudInit + linuxPrep := vm.Spec.Bootstrap.LinuxPrep + sysPrep := vm.Spec.Bootstrap.Sysprep + + if mtu := interfaceSpec.MTU; mtu != nil && cloudInit == nil { + allErrs = append(allErrs, field.Invalid( + interfacePath.Child("mtu"), + mtu, + "mtu is available only with the following bootstrap providers: CloudInit", + )) + } + + if routes := interfaceSpec.Routes; routes != nil && cloudInit == nil { + allErrs = append(allErrs, field.Invalid( + interfacePath.Child("routes"), + // Not exposing routes here in error message + "routes", + "routes is available only with the following bootstrap providers: CloudInit", + )) + } + + if nameservers := interfaceSpec.Nameservers; nameservers != nil { + sysprepNotAllowed := !lib.IsWindowsSysprepFSSEnabled() || sysPrep == nil || !equality.Semantic.DeepEqual(sysPrep.RawSysprep, corev1.SecretKeySelector{}) + if cloudInit == nil && linuxPrep == nil && sysprepNotAllowed { + allErrs = append(allErrs, field.Invalid( + interfacePath.Child("nameservers"), + strings.Join(nameservers, ","), + "nameservers is available only with the following bootstrap providers: CloudInit LinuxPrep and Sysprep (except for RawSysprep)", + )) + } + } + + if searchDomains := interfaceSpec.SearchDomains; searchDomains != nil { + sysprepNotAllowed := !lib.IsWindowsSysprepFSSEnabled() || sysPrep == nil || !equality.Semantic.DeepEqual(sysPrep.RawSysprep, corev1.SecretKeySelector{}) + if cloudInit == nil && linuxPrep == nil && sysprepNotAllowed { + allErrs = append(allErrs, field.Invalid( + interfacePath.Child("searchDomains"), + strings.Join(searchDomains, ","), + "searchDomains is available only with the following bootstrap providers: CloudInit LinuxPrep and Sysprep (except for RawSysprep)", + )) + } + } + + return allErrs +} + func (v validator) validateVolumes(ctx *context.WebhookRequestContext, vm *vmopv1.VirtualMachine) field.ErrorList { var allErrs field.ErrorList volumesPath := field.NewPath("spec", "volumes") diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go index aa9cee264..e7f6aa704 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" + "github.com/vmware-tanzu/vm-operator/api/v1alpha2/cloudinit" "github.com/vmware-tanzu/vm-operator/api/v1alpha2/common" pkgbuilder "github.com/vmware-tanzu/vm-operator/pkg/builder" "github.com/vmware-tanzu/vm-operator/pkg/lib" @@ -103,14 +104,6 @@ func unitTestsValidateCreate() { isWCPFaultDomainsFSSEnabled bool isInvalidAvailabilityZone bool isEmptyAvailabilityZone bool - isBootstrapCloudInit bool - isBootstrapCloudInitInline bool - isBootstrapLinuxPrep bool - isSysprepFeatureEnabled bool - isBootstrapSysPrep bool - isBootstrapSysPrepInline bool - isBootstrapVAppConfig bool - isBootstrapVAppConfigInline bool powerState vmopv1.VirtualMachinePowerState nextRestartTime string adminOnlyAnnotations bool @@ -219,45 +212,6 @@ func unitTestsValidateCreate() { ctx.vm.Labels[topology.KubernetesTopologyZoneLabelKey] = zoneName } - if args.isBootstrapCloudInit || args.isBootstrapCloudInitInline { - ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} - if args.isBootstrapCloudInit { - ctx.vm.Spec.Bootstrap.CloudInit.RawCloudConfig.Key = "cloud-init-key" - } - if args.isBootstrapCloudInitInline { - ctx.vm.Spec.Bootstrap.CloudInit.CloudConfig.Timezone = " dummy-tz" - } - } - if args.isBootstrapLinuxPrep { - ctx.vm.Spec.Bootstrap.LinuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} - } - if args.isSysprepFeatureEnabled { - Expect(os.Setenv(lib.WindowsSysprepFSS, "true")).To(Succeed()) - } - if args.isBootstrapSysPrep || args.isBootstrapSysPrepInline { - ctx.vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOff - ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} - if args.isBootstrapSysPrep { - ctx.vm.Spec.Bootstrap.Sysprep.RawSysprep.Key = "sysprep-key" - } - if args.isBootstrapSysPrepInline { - ctx.vm.Spec.Bootstrap.Sysprep.Sysprep.GUIRunOnce.Commands = []string{"hello"} - } - } - if args.isBootstrapVAppConfig || args.isBootstrapVAppConfigInline { - ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} - if args.isBootstrapVAppConfig { - ctx.vm.Spec.Bootstrap.VAppConfig.RawProperties = "some-vapp-prop" - } - if args.isBootstrapVAppConfigInline { - ctx.vm.Spec.Bootstrap.VAppConfig.Properties = []common.KeyValueOrSecretKeySelectorPair{ - { - Key: "key", - }, - } - } - } - if args.adminOnlyAnnotations { ctx.vm.Annotations[vmopv1.InstanceIDAnnotation] = updateSuffix ctx.vm.Annotations[vmopv1.FirstBootDoneAnnotation] = updateSuffix @@ -360,26 +314,6 @@ func unitTestsValidateCreate() { Entry("should deny when VM specifies invalid availability zone, there are no availability zones, and WCP FaultDomains FSS is enabled", createArgs{isInvalidAvailabilityZone: true, isNoAvailabilityZones: true, isWCPFaultDomainsFSSEnabled: true}, false, nil, nil), Entry("should deny when there are no availability zones and WCP FaultDomains FSS is enabled", createArgs{isNoAvailabilityZones: true, isWCPFaultDomainsFSSEnabled: true}, false, nil, nil), - Entry("should allow CloudInit", createArgs{isBootstrapCloudInit: true}, true, nil, nil), - Entry("should deny CloudInit with raw and inline", createArgs{isBootstrapCloudInit: true, isBootstrapCloudInitInline: true}, false, "cloudConfig and rawCloudConfig are mutually exclusive", nil), - Entry("should deny CloudInit with LinuxPrep", createArgs{isBootstrapCloudInit: true, isBootstrapLinuxPrep: true}, false, "CloudInit may not be used with any other bootstrap provider", nil), - Entry("should deny CloudInit with SysPrep", createArgs{isBootstrapCloudInit: true, isBootstrapSysPrep: true}, false, "CloudInit may not be used with any other bootstrap provider", nil), - Entry("should deny CloudInit with vApp", createArgs{isBootstrapCloudInit: true, isBootstrapVAppConfig: true}, false, "CloudInit may not be used with any other bootstrap provider", nil), - - Entry("should allow LinuxPrep", createArgs{isBootstrapLinuxPrep: true}, true, nil, nil), - Entry("should deny LinuxPrep with CloudInit", createArgs{isBootstrapLinuxPrep: true, isBootstrapCloudInit: true}, false, "LinuxPrep may not be used with either CloudInit or Sysprep bootstrap providers", nil), - Entry("should deny LinuxPrep with SysPrep", createArgs{isBootstrapLinuxPrep: true, isBootstrapSysPrep: true}, false, "LinuxPrep may not be used with either CloudInit or Sysprep bootstrap providers", nil), - - Entry("should allow sysprep when FSS is enabled", createArgs{isSysprepFeatureEnabled: true, isBootstrapSysPrep: true}, true, nil, nil), - Entry("should disallow sysprep when FSS is disabled", createArgs{isSysprepFeatureEnabled: false, isBootstrapSysPrep: true}, false, - field.Invalid(specPath.Child("bootstrap", "sysprep"), "sysprep", "the sysprep feature is not enabled").Error(), nil), - Entry("should deny sysprep with CloudInit", createArgs{isSysprepFeatureEnabled: true, isBootstrapSysPrep: true, isBootstrapCloudInit: true}, false, nil, nil), - - Entry("should allow vApp", createArgs{isBootstrapVAppConfig: true}, true, nil, nil), - Entry("should deny with raw and inline", createArgs{isBootstrapVAppConfig: true, isBootstrapVAppConfigInline: true}, false, "properties and rawProperties are mutually exclusive", nil), - Entry("should allow vApp with LinuxPrep", createArgs{isBootstrapVAppConfig: true, isBootstrapLinuxPrep: true}, true, nil, nil), - Entry("should allow vApp with SysPrep", createArgs{isBootstrapVAppConfig: true, isSysprepFeatureEnabled: true, isBootstrapSysPrep: true}, true, nil, nil), - Entry("should disallow creating VM with suspended power state", createArgs{powerState: vmopv1.VirtualMachinePowerStateSuspended}, false, field.Invalid(specPath.Child("powerState"), vmopv1.VirtualMachinePowerStateSuspended, "cannot set a new VM's power state to Suspended").Error(), nil), @@ -404,6 +338,182 @@ func unitTestsValidateCreate() { Entry("should allow creating VM with admin-only annotations set by WCP user when the Backup/Restore FSS is enabled", createArgs{adminOnlyAnnotations: true, isPrivilegedUser: true}, true, nil, nil), ) + Context("Bootstrap", func() { + type testParams struct { + setup func(ctx *unitValidatingWebhookContext) + validate func(response admission.Response) + expectAllowed bool + } + + doTest := func(args testParams) { + args.setup(ctx) + + var err error + ctx.WebhookRequestContext.Obj, err = builder.ToUnstructured(ctx.vm) + Expect(err).ToNot(HaveOccurred()) + + response := ctx.ValidateCreate(&ctx.WebhookRequestContext) + Expect(response.Allowed).To(Equal(args.expectAllowed)) + + if args.validate != nil { + args.validate(response) + } + } + + doValidateWithMsg := func(msgs ...string) func(admission.Response) { + return func(response admission.Response) { + reasons := string(response.Result.Reason) + for _, m := range msgs { + Expect(reasons).To(ContainSubstring(m)) + } + } + } + + DescribeTable("bootstrap create", doTest, + Entry("allow CloudInit bootstrap", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + }, + expectAllowed: true, + }, + ), + Entry("allow LinuxPrep bootstrap", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.LinuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} + }, + expectAllowed: true, + }, + ), + Entry("allow vAppConfig bootstrap", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} + }, + expectAllowed: true, + }, + ), + Entry("allow Sysprep bootstrap when WCP_Windows_Sysprep FSS is enabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + Expect(os.Setenv(lib.WindowsSysprepFSS, "true")).To(Succeed()) + ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + }, + expectAllowed: true, + }, + ), + Entry("disallow Sysprep bootstrap when WCP_Windows_Sysprep FSS is disabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + Expect(os.Setenv(lib.WindowsSysprepFSS, "false")).To(Succeed()) + ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + }, + validate: doValidateWithMsg( + `spec.bootstrap.sysprep: Invalid value: "Sysprep": the Sysprep feature is not enabled`, + ), + }, + ), + Entry("disallow CloudInit and LinuxPrep specified at the same time", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + ctx.vm.Spec.Bootstrap.LinuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} + }, + validate: doValidateWithMsg("CloudInit may not be used with any other bootstrap provider", + "LinuxPrep may not be used with either CloudInit or Sysprep bootstrap providers"), + }, + ), + Entry("disallow CloudInit and Sysprep specified at the same time", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + }, + validate: doValidateWithMsg("CloudInit may not be used with any other bootstrap provider"), + }, + ), + Entry("disallow CloudInit and vAppConfig specified at the same time", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} + }, + validate: doValidateWithMsg("CloudInit may not be used with any other bootstrap provider"), + }, + ), + Entry("disallow LinuxPrep and Sysprep specified at the same time", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.LinuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} + ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + }, + validate: doValidateWithMsg("LinuxPrep may not be used with either CloudInit or Sysprep bootstrap providers"), + }, + ), + Entry("allow LinuxPrep and vAppConfig specified at the same time", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.LinuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} + ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} + }, + expectAllowed: true, + }, + ), + Entry("allow Sysprep and vAppConfig specified at the same time when WCP_Windows_Sysprep FSS is enabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + Expect(os.Setenv(lib.WindowsSysprepFSS, "true")).To(Succeed()) + ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} + }, + expectAllowed: true, + }, + ), + Entry("disallow CloudInit mixing inline CloudConfig and RawCloudConfig", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{ + CloudConfig: cloudinit.CloudConfig{ + Timezone: "dummy-tz", + }, + RawCloudConfig: corev1.SecretKeySelector{ + Key: "cloud-init-key", + }, + } + }, + validate: doValidateWithMsg("cloudConfig and rawCloudConfig are mutually exclusive"), + }, + ), + Entry("disallow Sysprep mixing inline Sysprep and RawSysprep when FSS is enabled", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + Expect(os.Setenv(lib.WindowsSysprepFSS, "true")).To(Succeed()) + ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + ctx.vm.Spec.Bootstrap.Sysprep.Sysprep.GUIRunOnce.Commands = []string{"hello"} + ctx.vm.Spec.Bootstrap.Sysprep.RawSysprep.Key = "sysprep-key" + }, + validate: doValidateWithMsg("sysprep and rawSysprep are mutually exclusive"), + }, + ), + Entry("disallow vAppConfig mixing inline Properties and RawProperties", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: "key", + }, + }, + RawProperties: "some-vapp-prop", + } + }, + validate: doValidateWithMsg("properties and rawProperties are mutually exclusive"), + }, + ), + ) + }) + Context("Network", func() { type testParams struct { @@ -451,6 +561,31 @@ func unitTestsValidateCreate() { Entry("allow static", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + HostName: "my-vm", + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + Addresses: []string{ + "192.168.1.100/24", + "2605:a601:a0ba:720:2ce6:776d:8be4:2496/48", + }, + DHCP4: false, + DHCP6: false, + Gateway4: "192.168.1.1", + Gateway6: "2605:a601:a0ba:720:2ce6::1", + }, + }, + } + }, + expectAllowed: true, + }, + ), + + Entry("allow static mtu, nameservers, routes and searchDomains when bootstrap is CloudInit", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ HostName: "my-vm", Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ @@ -579,9 +714,53 @@ func unitTestsValidateCreate() { }, ), - Entry("validate nameservers", + // Please note mtu is available only with the following bootstrap providers: CloudInit + Entry("validate mtu when bootstrap doesn't support mtu", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.LinuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + HostName: "my-vm", + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + MTU: pointer.Int64(9000), + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.network.interfaces[0].mtu: Invalid value: 9000: mtu is available only with the following bootstrap providers: CloudInit`, + ), + }, + ), + + Entry("validate mtu when bootstrap supports mtu", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + ctx.vm.Spec.Network = vmopv1.VirtualMachineNetworkSpec{ + HostName: "my-vm", + Interfaces: []vmopv1.VirtualMachineNetworkInterfaceSpec{ + { + Name: "eth0", + MTU: pointer.Int64(9000), + }, + }, + } + }, + expectAllowed: true, + }, + ), + + // Please note nameservers is available only with the following bootstrap + // providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). + Entry("validate nameservers when bootstrap doesn't support nameservers", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + Expect(os.Setenv(lib.WindowsSysprepFSS, "true")).To(Succeed()) + ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + ctx.vm.Spec.Bootstrap.Sysprep.RawSysprep.Key = "sysprep-key" ctx.vm.Spec.Network.Interfaces[0].Nameservers = []string{ "not-an-ip", "192.168.1.1/24", @@ -590,13 +769,33 @@ func unitTestsValidateCreate() { validate: doValidateWithMsg( `spec.network.interfaces[0].nameservers[0]: Invalid value: "not-an-ip": must be an IPv4 or IPv6 address`, `spec.network.interfaces[0].nameservers[1]: Invalid value: "192.168.1.1/24": must be an IPv4 or IPv6 address`, + `spec.network.interfaces[0].nameservers: Invalid value: "not-an-ip,192.168.1.1/24": nameservers is available only with the following bootstrap providers: CloudInit LinuxPrep and Sysprep (except for RawSysprep)`, ), }, ), - Entry("validate routes", + Entry("validate nameservers when bootstrap supports nameservers", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + Expect(os.Setenv(lib.WindowsSysprepFSS, "true")).To(Succeed()) + ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + ctx.vm.Spec.Bootstrap.Sysprep.Sysprep.GUIRunOnce.Commands = []string{"hello"} + ctx.vm.Spec.Network.Interfaces[0].Nameservers = []string{ + "8.8.8.8", + "2001:4860:4860::8888", + } + }, + expectAllowed: true, + }, + ), + + // Please note routes is available only with the following bootstrap providers: CloudInit + Entry("validate routes when bootstrap doesn't support routes", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + Expect(os.Setenv(lib.WindowsSysprepFSS, "true")).To(Succeed()) + ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} + ctx.vm.Spec.Bootstrap.Sysprep.Sysprep.GUIRunOnce.Commands = []string{"hello"} ctx.vm.Spec.Network.Interfaces[0].Routes = []vmopv1.VirtualMachineNetworkRouteSpec{ { To: "10.100.10.1", @@ -617,10 +816,55 @@ func unitTestsValidateCreate() { `spec.network.interfaces[0].routes[0].via: Invalid value: "192.168.1": must be an IPv4 or IPv6 address`, `spec.network.interfaces[0].routes[1].via: Invalid value: "2463:foobar": must be an IPv4 or IPv6 address`, `spec.network.interfaces[0].routes[2]: Invalid value: "": cannot mix IP address families`, + `spec.network.interfaces[0].routes: Invalid value: "routes": routes is available only with the following bootstrap providers: CloudInit`, ), }, ), + Entry("validate routes when bootstrap supports routes", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} + ctx.vm.Spec.Network.Interfaces[0].Routes = []vmopv1.VirtualMachineNetworkRouteSpec{ + { + To: "10.100.10.1/24", + Via: "10.10.1.1", + Metric: 42, + }, + { + To: "fbd6:93e7:bc11:18b2:514f:2b1d:637a:f695/48", + Via: "ef71:6ce2:3b91:8349:b2b2:f76c:86ae:915b", + }, + } + }, + expectAllowed: true, + }, + ), + + // Please note this feature is available only with the following bootstrap + // providers: CloudInit, LinuxPrep, and Sysprep (except for RawSysprep). + Entry("validate searchDomains when bootstrap doesn't support searchDomains", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} + ctx.vm.Spec.Network.Interfaces[0].SearchDomains = []string{"dev.local"} + }, + validate: doValidateWithMsg( + `spec.network.interfaces[0].searchDomains: Invalid value: "dev.local": searchDomains is available only with the following bootstrap providers: CloudInit LinuxPrep and Sysprep (except for RawSysprep)`, + ), + }, + ), + + Entry("validate searchDomains when bootstrap supports searchDomains", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.LinuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} + ctx.vm.Spec.Network.Interfaces[0].SearchDomains = []string{"dev.local"} + }, + expectAllowed: true, + }, + ), + Entry("disallow creating VM with network interfaces resulting in a non-DNS1123 combined network interface CR name/label (`vmName-networkName-interfaceName`)", testParams{ setup: func(ctx *unitValidatingWebhookContext) { @@ -822,7 +1066,7 @@ func unitTestsValidateUpdate() { Entry("should allow sysprep when FSS is enabled", updateArgs{isSysprepFeatureEnabled: true, isSysprepTransportUsed: true}, true, nil, nil), Entry("should disallow sysprep when FSS is disabled", updateArgs{isSysprepFeatureEnabled: false, isSysprepTransportUsed: true}, false, - field.Invalid(field.NewPath("spec", "bootstrap", "sysprep"), "sysprep", "the sysprep feature is not enabled").Error(), nil), + field.Invalid(field.NewPath("spec", "bootstrap", "sysprep"), "Sysprep", "the Sysprep feature is not enabled").Error(), nil), Entry("should not error if sysprep FSS is disabled when sysprep is not used", updateArgs{isSysprepFeatureEnabled: false, isSysprepTransportUsed: false}, true, nil, nil), Entry("should allow updating suspended VM to powered on", updateArgs{oldPowerState: vmopv1.VirtualMachinePowerStateSuspended, newPowerState: vmopv1.VirtualMachinePowerStateOn}, true, From 20606549d0de87548d7f4ac15396a6648cf1b09f Mon Sep 17 00:00:00 2001 From: Sai Diliyaer Date: Mon, 6 Nov 2023 14:30:12 -0500 Subject: [PATCH 47/54] Move VM backup/restore related constants to api module --- api/v1alpha1/virtualmachine_types.go | 22 +++++++++++++++++++ api/v1alpha2/virtualmachine_types.go | 22 +++++++++++++++++++ .../providers/vsphere/constants/constants.go | 18 --------------- .../vsphere/session/session_vm_update.go | 4 ++-- .../vsphere/virtualmachine/backup.go | 17 +++++++------- .../vsphere/virtualmachine/backup_test.go | 21 +++++++++--------- .../vsphere/virtualmachine/configspec.go | 4 ++-- .../providers/vsphere2/constants/constants.go | 18 --------------- .../vsphere2/session/session_vm_update.go | 4 ++-- .../vsphere2/virtualmachine/backup.go | 17 +++++++------- .../vsphere2/virtualmachine/backup_test.go | 21 +++++++++--------- .../vsphere2/virtualmachine/configspec.go | 4 ++-- 12 files changed, 88 insertions(+), 84 deletions(-) diff --git a/api/v1alpha1/virtualmachine_types.go b/api/v1alpha1/virtualmachine_types.go index b562f50e7..05bd84393 100644 --- a/api/v1alpha1/virtualmachine_types.go +++ b/api/v1alpha1/virtualmachine_types.go @@ -112,6 +112,28 @@ const ( FirstBootDoneAnnotation = "virtualmachine." + GroupName + "/first-boot-done" ) +// VirtualMachine backup/restore related constants. +const ( + // ManagedByExtensionKey and ManagedByExtensionType represent the ManagedBy + // field on the VM. They are used to differentiate VM Service managed VMs + // from traditional vSphere VMs. + ManagedByExtensionKey = "com.vmware.vcenter.wcp" + ManagedByExtensionType = "VirtualMachine" + + // VMBackupKubeDataExtraConfigKey is the ExtraConfig key to persist the VM's + // Kubernetes resource spec data, compressed using gzip and base64-encoded. + VMBackupKubeDataExtraConfigKey = "vmservice.virtualmachine.kubedata" + // VMBackupBootstrapDataExtraConfigKey is the ExtraConfig key to persist the + // VM's bootstrap data object, compressed using gzip and base64-encoded. + VMBackupBootstrapDataExtraConfigKey = "vmservice.virtualmachine.bootstrapdata" + // VMBackupDiskDataExtraConfigKey is the ExtraConfig key to persist the VM's + // attached disk info in JSON, compressed using gzip and base64-encoded. + VMBackupDiskDataExtraConfigKey = "vmservice.virtualmachine.diskdata" + // VMBackupCloudInitInstanceIDExtraConfigKey is the ExtraConfig key to persist + // the VM's Cloud-Init instance ID, compressed using gzip and base64-encoded. + VMBackupCloudInitInstanceIDExtraConfigKey = "vmservice.virtualmachine.cloudinit.instanceid" +) + // VirtualMachinePort is unused and can be considered deprecated. type VirtualMachinePort struct { Port int `json:"port"` diff --git a/api/v1alpha2/virtualmachine_types.go b/api/v1alpha2/virtualmachine_types.go index 843253780..474d0c45b 100644 --- a/api/v1alpha2/virtualmachine_types.go +++ b/api/v1alpha2/virtualmachine_types.go @@ -109,6 +109,28 @@ const ( FirstBootDoneAnnotation = "virtualmachine." + GroupName + "/first-boot-done" ) +// VirtualMachine backup/restore related constants. +const ( + // ManagedByExtensionKey and ManagedByExtensionType represent the ManagedBy + // field on the VM. They are used to differentiate VM Service managed VMs + // from traditional vSphere VMs. + ManagedByExtensionKey = "com.vmware.vcenter.wcp" + ManagedByExtensionType = "VirtualMachine" + + // VMBackupKubeDataExtraConfigKey is the ExtraConfig key to persist the VM's + // Kubernetes resource spec data, compressed using gzip and base64-encoded. + VMBackupKubeDataExtraConfigKey = "vmservice.virtualmachine.kubedata" + // VMBackupBootstrapDataExtraConfigKey is the ExtraConfig key to persist the + // VM's bootstrap data object, compressed using gzip and base64-encoded. + VMBackupBootstrapDataExtraConfigKey = "vmservice.virtualmachine.bootstrapdata" + // VMBackupDiskDataExtraConfigKey is the ExtraConfig key to persist the VM's + // attached disk info in JSON, compressed using gzip and base64-encoded. + VMBackupDiskDataExtraConfigKey = "vmservice.virtualmachine.diskdata" + // VMBackupCloudInitInstanceIDExtraConfigKey is the ExtraConfig key to persist + // the VM's Cloud-Init instance ID, compressed using gzip and base64-encoded. + VMBackupCloudInitInstanceIDExtraConfigKey = "vmservice.virtualmachine.cloudinit.instanceid" +) + // VirtualMachinePowerState defines a VM's desired and observed power states. // +kubebuilder:validation:Enum=PoweredOff;PoweredOn;Suspended type VirtualMachinePowerState string diff --git a/pkg/vmprovider/providers/vsphere/constants/constants.go b/pkg/vmprovider/providers/vsphere/constants/constants.go index 838d7cc33..6205f4caf 100644 --- a/pkg/vmprovider/providers/vsphere/constants/constants.go +++ b/pkg/vmprovider/providers/vsphere/constants/constants.go @@ -16,11 +16,6 @@ const ( // VCVMAnnotation Annotation placed on the VM. VCVMAnnotation = "Virtual Machine managed by the vSphere Virtual Machine service" - // ManagedByExtensionKey and ManagedByExtensionType represent the ManagedBy field on the VM. - // Historically, this field was used to differentiate VM Service managed VMs from traditional ones. - ManagedByExtensionKey = "com.vmware.vcenter.wcp" - ManagedByExtensionType = "VirtualMachine" - // VSphereCustomizationBypassKey Annotation to skip applying VMware Tools Guest Customization. VSphereCustomizationBypassKey = pkg.VMOperatorKey + "/vsphere-customization" VSphereCustomizationBypassDisable = "disable" @@ -118,17 +113,4 @@ const ( V1alpha1SubnetMask = "V1alpha1_SubnetMask" // V1alpha1FormatNameservers is an alias for versioned templating function V1alpha1_FormatNameservers. V1alpha1FormatNameservers = "V1alpha1_FormatNameservers" - - // BackupVMKubeDataExtraConfigKey is the ExtraConfig key to the VirtualMachine - // resource's Kubernetes spec data, compressed using gzip and base64-encoded. - BackupVMKubeDataExtraConfigKey = "vmservice.virtualmachine.kubedata" - // BackupVMBootstrapDataExtraConfigKey is the ExtraConfig key to the VM's - // bootstrap data object, compressed using gzip and base64-encoded. - BackupVMBootstrapDataExtraConfigKey = "vmservice.virtualmachine.bootstrapdata" - // BackupVMDiskDataExtraConfigKey is the ExtraConfig key to the VM's disk info - // data in JSON, compressed using gzip and base64-encoded. - BackupVMDiskDataExtraConfigKey = "vmservice.virtualmachine.diskdata" - // BackupVMCloudInitInstanceIDExtraConfigKey is the ExtraConfig key to the VM's - // Cloud-Init instance ID, compressed using gzip and base64-encoded. - BackupVMCloudInitInstanceIDExtraConfigKey = "vmservice.virtualmachine.cloudinit.instanceid" ) diff --git a/pkg/vmprovider/providers/vsphere/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere/session/session_vm_update.go index be1cf70da..ba385f717 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere/session/session_vm_update.go @@ -422,8 +422,8 @@ func UpdateConfigSpecManagedBy( configSpec *vimTypes.VirtualMachineConfigSpec) { if config.ManagedBy == nil { configSpec.ManagedBy = &vimTypes.ManagedByInfo{ - ExtensionKey: constants.ManagedByExtensionKey, - Type: constants.ManagedByExtensionType, + ExtensionKey: vmopv1.ManagedByExtensionKey, + Type: vmopv1.ManagedByExtensionType, } } } diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go b/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go index 8ef722932..f84e9ad2b 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/backup.go @@ -15,7 +15,6 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/util" - "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" ) type VMDiskData struct { @@ -53,7 +52,7 @@ func BackupVirtualMachine(ctx context.BackupVirtualMachineContext) error { ctx.VMCtx.Logger.V(4).Info("Skipping VM kube data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ - Key: constants.BackupVMKubeDataExtraConfigKey, + Key: vmopv1.VMBackupKubeDataExtraConfigKey, Value: vmKubeDataBackup, }) } @@ -68,7 +67,7 @@ func BackupVirtualMachine(ctx context.BackupVirtualMachineContext) error { ctx.VMCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ - Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, + Key: vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey, Value: instanceIDBackup, }) } @@ -83,7 +82,7 @@ func BackupVirtualMachine(ctx context.BackupVirtualMachineContext) error { ctx.VMCtx.Logger.V(4).Info("Skipping VM bootstrap data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ - Key: constants.BackupVMBootstrapDataExtraConfigKey, + Key: vmopv1.VMBackupBootstrapDataExtraConfigKey, Value: bootstrapDataBackup, }) } @@ -98,7 +97,7 @@ func BackupVirtualMachine(ctx context.BackupVirtualMachineContext) error { ctx.VMCtx.Logger.V(4).Info("Skipping VM disk data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ - Key: constants.BackupVMDiskDataExtraConfigKey, + Key: vmopv1.VMBackupDiskDataExtraConfigKey, Value: diskDataBackup, }) } @@ -122,7 +121,7 @@ func getDesiredVMKubeDataForBackup( ecMap map[string]string) (string, error) { // If the ExtraConfig already contains the latest VM spec, determined by // 'metadata.generation', return an empty string to skip the backup. - if ecKubeData, ok := ecMap[constants.BackupVMKubeDataExtraConfigKey]; ok { + if ecKubeData, ok := ecMap[vmopv1.VMBackupKubeDataExtraConfigKey]; ok { vmFromBackup, err := constructVMObj(ecKubeData) if err != nil { return "", err @@ -158,7 +157,7 @@ func getDesiredCloudInitInstanceIDForBackup( ecMap map[string]string) (string, error) { // Cloud-Init instance ID should not be changed once persisted in VM's // ExtraConfig. Return an empty string to skip the backup if it exists. - if _, ok := ecMap[constants.BackupVMCloudInitInstanceIDExtraConfigKey]; ok { + if _, ok := ecMap[vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey]; ok { return "", nil } @@ -188,7 +187,7 @@ func getDesiredBootstrapDataForBackup( } // Return an empty string to skip the backup if the data is unchanged. - if bootstrapDataBackup == ecMap[constants.BackupVMBootstrapDataExtraConfigKey] { + if bootstrapDataBackup == ecMap[vmopv1.VMBackupBootstrapDataExtraConfigKey] { return "", nil } @@ -233,7 +232,7 @@ func getDesiredDiskDataForBackup( } // Return an empty string to skip the backup if the data is unchanged. - if diskDataBackup == ecMap[constants.BackupVMDiskDataExtraConfigKey] { + if diskDataBackup == ecMap[vmopv1.VMBackupDiskDataExtraConfigKey] { return "", nil } diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go b/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go index fbe47f966..9b8bd19c4 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/backup_test.go @@ -18,7 +18,6 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/util" - "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/virtualmachine" "github.com/vmware-tanzu/vm-operator/test/builder" ) @@ -75,7 +74,7 @@ func backupTests() { _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ ExtraConfig: []types.BaseOptionValue{ &types.OptionValue{ - Key: constants.BackupVMKubeDataExtraConfigKey, + Key: vmopv1.VMBackupKubeDataExtraConfigKey, Value: backupVMYamlEncoded, }, }, @@ -97,7 +96,7 @@ func backupTests() { vmCopy.Status = vmopv1.VirtualMachineStatus{} vmCopyYaml, err := yaml.Marshal(vmCopy) Expect(err).NotTo(HaveOccurred()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, string(vmCopyYaml)) + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupKubeDataExtraConfigKey, string(vmCopyYaml)) }) }) @@ -116,7 +115,7 @@ func backupTests() { _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ ExtraConfig: []types.BaseOptionValue{ &types.OptionValue{ - Key: constants.BackupVMKubeDataExtraConfigKey, + Key: vmopv1.VMBackupKubeDataExtraConfigKey, Value: encodedKubeDataBackup, }, }, @@ -134,7 +133,7 @@ func backupTests() { DiskUUIDToPVC: nil, } Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, kubeDataBackup) + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupKubeDataExtraConfigKey, kubeDataBackup) }) }) }) @@ -153,7 +152,7 @@ func backupTests() { bootstrapDataJSON, err := json.Marshal(bootstrapDataRaw) Expect(err).NotTo(HaveOccurred()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMBootstrapDataExtraConfigKey, string(bootstrapDataJSON)) + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupBootstrapDataExtraConfigKey, string(bootstrapDataJSON)) }) }) @@ -169,7 +168,7 @@ func backupTests() { DiskUUIDToPVC: nil, } Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, "") + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupDiskDataExtraConfigKey, "") }) }) @@ -198,7 +197,7 @@ func backupTests() { } diskDataJSON, err := json.Marshal(diskData) Expect(err).NotTo(HaveOccurred()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, string(diskDataJSON)) + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupDiskDataExtraConfigKey, string(diskDataJSON)) }) }) }) @@ -215,7 +214,7 @@ func backupTests() { _, err := vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ ExtraConfig: []types.BaseOptionValue{ &types.OptionValue{ - Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, + Key: vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey, Value: "ec-instance-id", }, }, @@ -234,7 +233,7 @@ func backupTests() { DiskUUIDToPVC: nil, } Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "ec-instance-id") + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey, "ec-instance-id") }) }) @@ -255,7 +254,7 @@ func backupTests() { DiskUUIDToPVC: nil, } Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "annotation-instance-id") + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey, "annotation-instance-id") }) }) diff --git a/pkg/vmprovider/providers/vsphere/virtualmachine/configspec.go b/pkg/vmprovider/providers/vsphere/virtualmachine/configspec.go index 7ff2d6ab6..c9bd92264 100644 --- a/pkg/vmprovider/providers/vsphere/virtualmachine/configspec.go +++ b/pkg/vmprovider/providers/vsphere/virtualmachine/configspec.go @@ -41,8 +41,8 @@ func CreateConfigSpec( configSpec.MemoryMB = MemoryQuantityToMb(vmClassSpec.Hardware.Memory) configSpec.ManagedBy = &vimtypes.ManagedByInfo{ - ExtensionKey: constants.ManagedByExtensionKey, - Type: constants.ManagedByExtensionType, + ExtensionKey: vmopv1.ManagedByExtensionKey, + Type: vmopv1.ManagedByExtensionType, } // Populate the CPU reservation and limits in the ConfigSpec if VAPI fields specify any. diff --git a/pkg/vmprovider/providers/vsphere2/constants/constants.go b/pkg/vmprovider/providers/vsphere2/constants/constants.go index 725b1ad0c..289522810 100644 --- a/pkg/vmprovider/providers/vsphere2/constants/constants.go +++ b/pkg/vmprovider/providers/vsphere2/constants/constants.go @@ -16,11 +16,6 @@ const ( // VCVMAnnotation Annotation placed on the VM. VCVMAnnotation = "Virtual Machine managed by the vSphere Virtual Machine service" - // ManagedByExtensionKey and ManagedByExtensionType represent the ManagedBy field on the VM. - // Historically, this field was used to differentiate VM Service managed VMs from traditional ones. - ManagedByExtensionKey = "com.vmware.vcenter.wcp" - ManagedByExtensionType = "VirtualMachine" - // VSphereCustomizationBypassKey Annotation to skip applying VMware Tools Guest Customization. VSphereCustomizationBypassKey = pkg.VMOperatorKey + "/vsphere-customization" VSphereCustomizationBypassDisable = "disable" @@ -134,17 +129,4 @@ const ( V1alpha2SubnetMask = "V1alpha2_SubnetMask" // V1alpha2FormatNameservers is an alias for versioned templating function V1alpha2_FormatNameservers. V1alpha2FormatNameservers = "V1alpha2_FormatNameservers" - - // BackupVMKubeDataExtraConfigKey is the ExtraConfig key to the VirtualMachine - // resource's Kubernetes spec data, compressed using gzip and base64-encoded. - BackupVMKubeDataExtraConfigKey = "vmservice.virtualmachine.kubedata" - // BackupVMBootstrapDataExtraConfigKey is the ExtraConfig key to the VM's - // bootstrap data object, compressed using gzip and base64-encoded. - BackupVMBootstrapDataExtraConfigKey = "vmservice.virtualmachine.bootstrapdata" - // BackupVMDiskDataExtraConfigKey is the ExtraConfig key to the VM's disk info - // data in JSON, compressed using gzip and base64-encoded. - BackupVMDiskDataExtraConfigKey = "vmservice.virtualmachine.diskdata" - // BackupVMCloudInitInstanceIDExtraConfigKey is the ExtraConfig key to the VM's - // Cloud-Init instance ID, compressed using gzip and base64-encoded. - BackupVMCloudInitInstanceIDExtraConfigKey = "vmservice.virtualmachine.cloudinit.instanceid" ) diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go index 6a715ab92..23f1b8295 100644 --- a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go @@ -417,8 +417,8 @@ func UpdateConfigSpecManagedBy( configSpec *vimTypes.VirtualMachineConfigSpec) { if config.ManagedBy == nil { configSpec.ManagedBy = &vimTypes.ManagedByInfo{ - ExtensionKey: constants.ManagedByExtensionKey, - Type: constants.ManagedByExtensionType, + ExtensionKey: vmopv1.ManagedByExtensionKey, + Type: vmopv1.ManagedByExtensionType, } } } diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go index 2f5f3b7a9..688eee8c2 100644 --- a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup.go @@ -15,7 +15,6 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/util" - "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" ) type VMDiskData struct { @@ -53,7 +52,7 @@ func BackupVirtualMachine(ctx context.BackupVirtualMachineContextA2) error { ctx.VMCtx.Logger.V(4).Info("Skipping VM kube data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ - Key: constants.BackupVMKubeDataExtraConfigKey, + Key: vmopv1.VMBackupKubeDataExtraConfigKey, Value: vmKubeDataBackup, }) } @@ -68,7 +67,7 @@ func BackupVirtualMachine(ctx context.BackupVirtualMachineContextA2) error { ctx.VMCtx.Logger.V(4).Info("Skipping cloud-init instance ID as already stored") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ - Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, + Key: vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey, Value: instanceIDBackup, }) } @@ -83,7 +82,7 @@ func BackupVirtualMachine(ctx context.BackupVirtualMachineContextA2) error { ctx.VMCtx.Logger.V(4).Info("Skipping VM bootstrap data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ - Key: constants.BackupVMBootstrapDataExtraConfigKey, + Key: vmopv1.VMBackupBootstrapDataExtraConfigKey, Value: bootstrapDataBackup, }) } @@ -98,7 +97,7 @@ func BackupVirtualMachine(ctx context.BackupVirtualMachineContextA2) error { ctx.VMCtx.Logger.V(4).Info("Skipping VM disk data backup as unchanged") } else { ecToUpdate = append(ecToUpdate, &types.OptionValue{ - Key: constants.BackupVMDiskDataExtraConfigKey, + Key: vmopv1.VMBackupDiskDataExtraConfigKey, Value: diskDataBackup, }) } @@ -122,7 +121,7 @@ func getDesiredVMKubeDataForBackup( ecMap map[string]string) (string, error) { // If the ExtraConfig already contains the latest VM spec, determined by // 'metadata.generation', return an empty string to skip the backup. - if ecKubeData, ok := ecMap[constants.BackupVMKubeDataExtraConfigKey]; ok { + if ecKubeData, ok := ecMap[vmopv1.VMBackupKubeDataExtraConfigKey]; ok { vmFromBackup, err := constructVMObj(ecKubeData) if err != nil { return "", err @@ -158,7 +157,7 @@ func getDesiredCloudInitInstanceIDForBackup( ecMap map[string]string) (string, error) { // Cloud-Init instance ID should not be changed once persisted in VM's // ExtraConfig. Return an empty string to skip the backup if it exists. - if _, ok := ecMap[constants.BackupVMCloudInitInstanceIDExtraConfigKey]; ok { + if _, ok := ecMap[vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey]; ok { return "", nil } @@ -188,7 +187,7 @@ func getDesiredBootstrapDataForBackup( } // Return an empty string to skip the backup if the data is unchanged. - if bootstrapDataBackup == ecMap[constants.BackupVMBootstrapDataExtraConfigKey] { + if bootstrapDataBackup == ecMap[vmopv1.VMBackupBootstrapDataExtraConfigKey] { return "", nil } @@ -233,7 +232,7 @@ func getDesiredDiskDataForBackup( } // Return an empty string to skip the backup if the data is unchanged. - if diskDataBackup == ecMap[constants.BackupVMDiskDataExtraConfigKey] { + if diskDataBackup == ecMap[vmopv1.VMBackupDiskDataExtraConfigKey] { return "", nil } diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go index 2a4cbcf98..dd47c344d 100644 --- a/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/backup_test.go @@ -18,7 +18,6 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2" "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/util" - "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/constants" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" "github.com/vmware-tanzu/vm-operator/test/builder" ) @@ -75,7 +74,7 @@ func backupTests() { _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ ExtraConfig: []types.BaseOptionValue{ &types.OptionValue{ - Key: constants.BackupVMKubeDataExtraConfigKey, + Key: vmopv1.VMBackupKubeDataExtraConfigKey, Value: backupVMYamlEncoded, }, }, @@ -97,7 +96,7 @@ func backupTests() { vmCopy.Status = vmopv1.VirtualMachineStatus{} vmCopyYaml, err := yaml.Marshal(vmCopy) Expect(err).NotTo(HaveOccurred()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, string(vmCopyYaml)) + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupKubeDataExtraConfigKey, string(vmCopyYaml)) }) }) @@ -116,7 +115,7 @@ func backupTests() { _, err = vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ ExtraConfig: []types.BaseOptionValue{ &types.OptionValue{ - Key: constants.BackupVMKubeDataExtraConfigKey, + Key: vmopv1.VMBackupKubeDataExtraConfigKey, Value: encodedKubeDataBackup, }, }, @@ -134,7 +133,7 @@ func backupTests() { DiskUUIDToPVC: nil, } Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMKubeDataExtraConfigKey, kubeDataBackup) + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupKubeDataExtraConfigKey, kubeDataBackup) }) }) }) @@ -153,7 +152,7 @@ func backupTests() { bootstrapDataJSON, err := json.Marshal(bootstrapDataRaw) Expect(err).NotTo(HaveOccurred()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMBootstrapDataExtraConfigKey, string(bootstrapDataJSON)) + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupBootstrapDataExtraConfigKey, string(bootstrapDataJSON)) }) }) @@ -169,7 +168,7 @@ func backupTests() { DiskUUIDToPVC: nil, } Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, "") + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupDiskDataExtraConfigKey, "") }) }) @@ -198,7 +197,7 @@ func backupTests() { } diskDataJSON, err := json.Marshal(diskData) Expect(err).NotTo(HaveOccurred()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMDiskDataExtraConfigKey, string(diskDataJSON)) + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupDiskDataExtraConfigKey, string(diskDataJSON)) }) }) }) @@ -215,7 +214,7 @@ func backupTests() { _, err := vcVM.Reconfigure(vmCtx, types.VirtualMachineConfigSpec{ ExtraConfig: []types.BaseOptionValue{ &types.OptionValue{ - Key: constants.BackupVMCloudInitInstanceIDExtraConfigKey, + Key: vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey, Value: "ec-instance-id", }, }, @@ -234,7 +233,7 @@ func backupTests() { DiskUUIDToPVC: nil, } Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "ec-instance-id") + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey, "ec-instance-id") }) }) @@ -255,7 +254,7 @@ func backupTests() { DiskUUIDToPVC: nil, } Expect(virtualmachine.BackupVirtualMachine(backupVMCtx)).To(Succeed()) - verifyBackupDataInExtraConfig(ctx, vcVM, constants.BackupVMCloudInitInstanceIDExtraConfigKey, "annotation-instance-id") + verifyBackupDataInExtraConfig(ctx, vcVM, vmopv1.VMBackupCloudInitInstanceIDExtraConfigKey, "annotation-instance-id") }) }) diff --git a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go index b61a865b9..575849d39 100644 --- a/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go +++ b/pkg/vmprovider/providers/vsphere2/virtualmachine/configspec.go @@ -43,8 +43,8 @@ func CreateConfigSpec( configSpec.NumCPUs = int32(vmClassSpec.Hardware.Cpus) configSpec.MemoryMB = MemoryQuantityToMb(vmClassSpec.Hardware.Memory) configSpec.ManagedBy = &types.ManagedByInfo{ - ExtensionKey: constants.ManagedByExtensionKey, - Type: constants.ManagedByExtensionType, + ExtensionKey: vmopv1.ManagedByExtensionKey, + Type: vmopv1.ManagedByExtensionType, } if val, ok := vmCtx.VM.Annotations[constants.FirmwareOverrideAnnotation]; ok && (val == "efi" || val == "bios") { From a9ca59b0cb3a815bc4b48c2d82e24039cc880c2f Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 8 Nov 2023 12:42:08 -0600 Subject: [PATCH 48/54] Revert "Mark v1a2 VirtualMachineClass as namespace scoped" This reverts commit 25b6933d95656b4178f23a5a7c4229c0cf99573b. While apparently very biased towards v1a1, the controller-gen manifest output is not deterministic. Since this change was just for informational purposes - the Scope field is fixed up later - revert this change. --- api/v1alpha2/virtualmachineclass_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1alpha2/virtualmachineclass_types.go b/api/v1alpha2/virtualmachineclass_types.go index c9ada2031..a53500b98 100644 --- a/api/v1alpha2/virtualmachineclass_types.go +++ b/api/v1alpha2/virtualmachineclass_types.go @@ -244,7 +244,7 @@ type VirtualMachineClassStatus struct { } // +kubebuilder:object:root=true -// +kubebuilder:resource:scope=Namespaced,shortName=vmclass +// +kubebuilder:resource:scope=Cluster,shortName=vmclass // +kubebuilder:storageversion // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="CPU",type="string",JSONPath=".spec.hardware.cpus" From 5fed8a3c016886f17c947e6594ebaa5d07113908 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 8 Nov 2023 13:33:22 -0600 Subject: [PATCH 49/54] Add missing omitempty on CloudConfig User Passwd field This was just accidentally omitted and required for proper serialization. --- api/v1alpha2/cloudinit/cloudconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1alpha2/cloudinit/cloudconfig.go b/api/v1alpha2/cloudinit/cloudconfig.go index 6f2baae58..60d37cc29 100644 --- a/api/v1alpha2/cloudinit/cloudconfig.go +++ b/api/v1alpha2/cloudinit/cloudconfig.go @@ -130,7 +130,7 @@ type User struct { // please use HashedPasswd instead. // // +optional - Passwd *corev1.SecretKeySelector `json:"passwd"` + Passwd *corev1.SecretKeySelector `json:"passwd,omitempty"` // PrimaryGroup is the primary group for the user. // From b71e0633cb5d9c854344c9a5e76179b048b0003d Mon Sep 17 00:00:00 2001 From: Sagar Muchhal Date: Wed, 8 Nov 2023 12:32:43 -0800 Subject: [PATCH 50/54] Removing duplicate error logger The same error during MV reconcile normal loop gets logged twice which clutters the log files with duplicates. Removing one of the loggers from VM reconcile loop. Signed-off-by: Sagar Muchhal --- controllers/virtualmachine/v1alpha1/virtualmachine_controller.go | 1 - controllers/virtualmachine/v1alpha2/virtualmachine_controller.go | 1 - 2 files changed, 2 deletions(-) diff --git a/controllers/virtualmachine/v1alpha1/virtualmachine_controller.go b/controllers/virtualmachine/v1alpha1/virtualmachine_controller.go index a91f95929..5bda8e4d6 100644 --- a/controllers/virtualmachine/v1alpha1/virtualmachine_controller.go +++ b/controllers/virtualmachine/v1alpha1/virtualmachine_controller.go @@ -462,7 +462,6 @@ func (r *Reconciler) ReconcileNormal(ctx *context.VirtualMachineContext) (reterr }() if err := r.VMProvider.CreateOrUpdateVirtualMachine(ctx, ctx.VM); err != nil { - ctx.Logger.Error(err, "Failed to reconcile VirtualMachine") r.Recorder.EmitEvent(ctx.VM, "CreateOrUpdate", err, false) return err } diff --git a/controllers/virtualmachine/v1alpha2/virtualmachine_controller.go b/controllers/virtualmachine/v1alpha2/virtualmachine_controller.go index ceaea80f9..065d38ec3 100644 --- a/controllers/virtualmachine/v1alpha2/virtualmachine_controller.go +++ b/controllers/virtualmachine/v1alpha2/virtualmachine_controller.go @@ -328,7 +328,6 @@ func (r *Reconciler) ReconcileNormal(ctx *context.VirtualMachineContextA2) (rete }() if err := r.VMProvider.CreateOrUpdateVirtualMachine(ctx, ctx.VM); err != nil { - ctx.Logger.Error(err, "Failed to reconcile VirtualMachine") r.Recorder.EmitEvent(ctx.VM, "CreateOrUpdate", err, false) return err } From e547347247accefa46eb08f6de0d0dbb473feeb3 Mon Sep 17 00:00:00 2001 From: Bryan Venteicher Date: Wed, 8 Nov 2023 16:49:04 -0600 Subject: [PATCH 51/54] Make VM Spec BootDiskCapacity field a pointer This field is a resource.Quantity which when not set still outputs as "0". This field is not likely commonly used, and outputting of a zero size is potentially confusing or misleading. Also, in the k8s API, resource.Quantity fields are always a pointer. While here, remove this field defaulting in the version conversion fuzzer since that is now supported after f65ef245. --- api/v1alpha1/conversion_test.go | 2 -- api/v1alpha1/virtualmachine_conversion.go | 12 ++++++------ api/v1alpha2/virtualmachine_types.go | 2 +- api/v1alpha2/zz_generated.deepcopy.go | 6 +++++- .../providers/vsphere2/session/session_vm.go | 2 +- .../providers/vsphere2/vmlifecycle/create_clone.go | 2 +- pkg/vmprovider/providers/vsphere2/vmprovider_vm.go | 2 +- .../providers/vsphere2/vmprovider_vm_test.go | 6 +++--- .../v1alpha2/validation/virtualmachine_validator.go | 6 +++--- 9 files changed, 21 insertions(+), 19 deletions(-) diff --git a/api/v1alpha1/conversion_test.go b/api/v1alpha1/conversion_test.go index 92a88adfc..1957487be 100644 --- a/api/v1alpha1/conversion_test.go +++ b/api/v1alpha1/conversion_test.go @@ -9,7 +9,6 @@ import ( fuzz "github.com/google/gofuzz" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/api/apitesting/fuzzer" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" @@ -125,7 +124,6 @@ func overrideVirtualMachineFieldsFuncs(codecs runtimeserializer.CodecFactory) [] vmSpec.ReadinessGates = nil vmSpec.ReadinessProbe.GuestInfo = nil - vmSpec.Advanced.BootDiskCapacity = resource.Quantity{} }, func(vmStatus *v1alpha1.VirtualMachineStatus, c fuzz.Continue) { c.Fuzz(vmStatus) diff --git a/api/v1alpha1/virtualmachine_conversion.go b/api/v1alpha1/virtualmachine_conversion.go index dc1253e09..3aed9f967 100644 --- a/api/v1alpha1/virtualmachine_conversion.go +++ b/api/v1alpha1/virtualmachine_conversion.go @@ -354,7 +354,7 @@ func convert_v1alpha1_VirtualMachineAdvancedOptions_To_v1alpha2_VirtualMachineAd return out } -func convert_v1alpha1_VsphereVolumes_To_v1alpah2_BootDiskCapacity(volumes []VirtualMachineVolume) resource.Quantity { +func convert_v1alpha1_VsphereVolumes_To_v1alpah2_BootDiskCapacity(volumes []VirtualMachineVolume) *resource.Quantity { // The v1a1 VsphereVolume was never a great API as you had to know the DeviceKey upfront; at the time our // API was private - only used by CAPW - and predates the "VM Service" VMs; In v1a2, we only support resizing // the boot disk via an explicit field. As good as we can here, map v1a1 volume into the v1a2 specific field. @@ -365,13 +365,13 @@ func convert_v1alpha1_VsphereVolumes_To_v1alpah2_BootDiskCapacity(volumes []Virt if vsVol != nil && vsVol.DeviceKey != nil && *vsVol.DeviceKey == bootDiskDeviceKey { // This VsphereVolume has the well-known boot disk device key. Return that capacity if set. if capacity := vsVol.Capacity.StorageEphemeral(); capacity != nil { - return *capacity + return capacity } break } } - return resource.Quantity{} + return nil } func convert_v1alpha2_VirtualMachineAdvancedSpec_To_v1alpha1_VirtualMachineAdvancedOptions( @@ -405,8 +405,8 @@ func convert_v1alpha2_VirtualMachineAdvancedSpec_To_v1alpha1_VirtualMachineAdvan return out } -func convert_v1alpha2_BootDiskCapacity_To_v1alpha1_VirtualMachineVolume(capacity resource.Quantity) *VirtualMachineVolume { - if capacity.IsZero() { +func convert_v1alpha2_BootDiskCapacity_To_v1alpha1_VirtualMachineVolume(capacity *resource.Quantity) *VirtualMachineVolume { + if capacity == nil || capacity.IsZero() { return nil } @@ -416,7 +416,7 @@ func convert_v1alpha2_BootDiskCapacity_To_v1alpha1_VirtualMachineVolume(capacity Name: name, VsphereVolume: &VsphereVolumeSource{ Capacity: corev1.ResourceList{ - corev1.ResourceEphemeralStorage: capacity, + corev1.ResourceEphemeralStorage: *capacity, }, DeviceKey: pointer.Int(bootDiskDeviceKey), }, diff --git a/api/v1alpha2/virtualmachine_types.go b/api/v1alpha2/virtualmachine_types.go index 474d0c45b..c1c44a665 100644 --- a/api/v1alpha2/virtualmachine_types.go +++ b/api/v1alpha2/virtualmachine_types.go @@ -384,7 +384,7 @@ type VirtualMachineAdvancedSpec struct { // affect the VM. // // +optional - BootDiskCapacity resource.Quantity `json:"bootDiskCapacity,omitempty"` + BootDiskCapacity *resource.Quantity `json:"bootDiskCapacity,omitempty"` // DefaultVolumeProvisioningMode specifies the default provisioning mode for // persistent volumes managed by this VM. diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index 780aaccf7..f7c4eb4e3 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -414,7 +414,11 @@ func (in *VirtualMachine) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineAdvancedSpec) DeepCopyInto(out *VirtualMachineAdvancedSpec) { *out = *in - out.BootDiskCapacity = in.BootDiskCapacity.DeepCopy() + if in.BootDiskCapacity != nil { + in, out := &in.BootDiskCapacity, &out.BootDiskCapacity + x := (*in).DeepCopy() + *out = &x + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineAdvancedSpec. diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm.go b/pkg/vmprovider/providers/vsphere2/session/session_vm.go index a7aaff105..cd074b3de 100644 --- a/pkg/vmprovider/providers/vsphere2/session/session_vm.go +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm.go @@ -17,7 +17,7 @@ func updateVirtualDiskDeviceChanges( virtualDisks object.VirtualDeviceList) ([]vimTypes.BaseVirtualDeviceConfigSpec, error) { capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity - if capacity.IsZero() { + if capacity == nil || capacity.IsZero() { return nil, nil } diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go index 299d0bba7..9a43a36ec 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/create_clone.go @@ -160,7 +160,7 @@ func resizeBootDiskDeviceChange( virtualDisks object.VirtualDeviceList) []vimtypes.BaseVirtualDeviceConfigSpec { capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity - if capacity.IsZero() { + if capacity == nil || capacity.IsZero() { return nil } diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go index ab05c5d30..8ca079b52 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go @@ -945,7 +945,7 @@ func (vs *vSphereVMProvider) vmCreateGenConfigSpecChangeBootDiskSize( _ *VMCreateArgs) error { capacity := vmCtx.VM.Spec.Advanced.BootDiskCapacity - if capacity.IsZero() { + if capacity == nil || capacity.IsZero() { return nil } diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go index 19a2c15a7..c6e5f228d 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go @@ -1507,10 +1507,10 @@ func vmTests() { }) Context("Should resize root disk", func() { - newSize := resource.MustParse("4242Gi") - It("Succeeds", func() { - vm.Spec.Advanced.BootDiskCapacity = newSize + newSize := resource.MustParse("4242Gi") + + vm.Spec.Advanced.BootDiskCapacity = &newSize vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOn vcVM, err := createOrUpdateAndGetVcVM(ctx, vm) Expect(err).ToNot(HaveOccurred()) diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index bd610f3e6..857126444 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -612,10 +612,10 @@ func (v validator) validateAdvanced(ctx *context.WebhookRequestContext, vm *vmop advancedPath := field.NewPath("spec", "advanced") advanced := &vm.Spec.Advanced - if diskCap := advanced.BootDiskCapacity; !diskCap.IsZero() { - if diskCap.Value()%megaByte.Value() != 0 { + if capacity := advanced.BootDiskCapacity; capacity != nil && !capacity.IsZero() { + if capacity.Value()%megaByte.Value() != 0 { allErrs = append(allErrs, field.Invalid(advancedPath.Child("bootDiskCapacity"), - diskCap.Value(), vSphereVolumeSizeNotMBMultiple)) + capacity.Value(), vSphereVolumeSizeNotMBMultiple)) } } From 92acdd6b8c52a215b2b3ca664fcd8a64dd490b53 Mon Sep 17 00:00:00 2001 From: Sreyas Natarajan <53065832+sreyasn@users.noreply.github.com> Date: Mon, 13 Nov 2023 10:24:08 -0800 Subject: [PATCH 52/54] Remove updates to DeviceGroups (#270) This change removes the update of deviceGroups in the UpdateVM path. We don't currently support this workflow on Update and DeviceGroups are already covered in the CreatePath. The code for this currently incomplete to support deviceGroups on update and will be prioritized when VM Operator supports resizing VMs. --- .../vsphere/session/session_vm_update.go | 14 ----- .../vsphere/session/session_vm_update_test.go | 53 ------------------- .../vsphere2/session/session_vm_update.go | 14 ----- .../session/session_vm_update_test.go | 53 ------------------- 4 files changed, 134 deletions(-) diff --git a/pkg/vmprovider/providers/vsphere/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere/session/session_vm_update.go index ba385f717..7138128f7 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere/session/session_vm_update.go @@ -440,19 +440,6 @@ func UpdateConfigSpecFirmware( } } -// UpdateConfigSpecDeviceGroups sets the desired config spec device groups to reconcile by differencing the -// current VM config and the class config spec device groups. -func UpdateConfigSpecDeviceGroups( - config *vimTypes.VirtualMachineConfigInfo, - configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec) { - - if classConfigSpec.DeviceGroups != nil { - if config.DeviceGroups == nil || !reflect.DeepEqual(classConfigSpec.DeviceGroups.DeviceGroup, config.DeviceGroups.DeviceGroup) { - configSpec.DeviceGroups = classConfigSpec.DeviceGroups - } - } -} - // updateConfigSpec overlays the VM Class spec with the provided ConfigSpec to form a desired // ConfigSpec that will be used to reconfigure the VM. func updateConfigSpec( @@ -480,7 +467,6 @@ func updateConfigSpec( vmCtx.VM, updateArgs.ExtraConfig, updateArgs.VirtualMachineImageV1Alpha1Compatible) UpdateConfigSpecChangeBlockTracking(config, configSpec, updateArgs.ConfigSpec, vmCtx.VM.Spec) UpdateConfigSpecFirmware(config, configSpec, vmCtx.VM) - UpdateConfigSpecDeviceGroups(config, configSpec, updateArgs.ConfigSpec) return configSpec } diff --git a/pkg/vmprovider/providers/vsphere/session/session_vm_update_test.go b/pkg/vmprovider/providers/vsphere/session/session_vm_update_test.go index 116f3687d..2c3e0d078 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_vm_update_test.go +++ b/pkg/vmprovider/providers/vsphere/session/session_vm_update_test.go @@ -807,59 +807,6 @@ var _ = Describe("Update ConfigSpec", func() { }) - Context("DeviceGroups", func() { - var classConfigSpec *vimTypes.VirtualMachineConfigSpec - - BeforeEach(func() { - classConfigSpec = &vimTypes.VirtualMachineConfigSpec{} - }) - - It("No DeviceGroups set in class config spec", func() { - session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) - Expect(configSpec.DeviceGroups).To(BeNil()) - }) - - It("DeviceGroups set in class config spec", func() { - classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ - DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ - &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ - GroupInstanceKey: int32(400), - }, - }, - } - - session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) - Expect(configSpec.DeviceGroups).NotTo(BeNil()) - Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) - deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() - Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) - }) - - It("configInfo DeviceGroups set with vals different than the class config spec", func() { - classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ - DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ - &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ - GroupInstanceKey: int32(400), - }, - }, - } - - config.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ - DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ - &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ - GroupInstanceKey: int32(500), - }, - }, - } - - session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) - Expect(configSpec.DeviceGroups).NotTo(BeNil()) - Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) - deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() - Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) - }) - }) - Context("Ethernet Card Changes", func() { var expectedList object.VirtualDeviceList var currentList object.VirtualDeviceList diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go index 23f1b8295..ab9172d2e 100644 --- a/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update.go @@ -435,19 +435,6 @@ func UpdateConfigSpecFirmware( } } -// UpdateConfigSpecDeviceGroups sets the desired config spec device groups to reconcile by differencing the -// current VM config and the class config spec device groups. -func UpdateConfigSpecDeviceGroups( - config *vimTypes.VirtualMachineConfigInfo, - configSpec, classConfigSpec *vimTypes.VirtualMachineConfigSpec) { - - if classConfigSpec.DeviceGroups != nil { - if config.DeviceGroups == nil || !reflect.DeepEqual(classConfigSpec.DeviceGroups.DeviceGroup, config.DeviceGroups.DeviceGroup) { - configSpec.DeviceGroups = classConfigSpec.DeviceGroups - } - } -} - // updateConfigSpec overlays the VM Class spec with the provided ConfigSpec to form a desired // ConfigSpec that will be used to reconfigure the VM. func updateConfigSpec( @@ -475,7 +462,6 @@ func updateConfigSpec( vmCtx.VM, updateArgs.ExtraConfig, updateArgs.VirtualMachineImageV1Alpha1Compatible) UpdateConfigSpecChangeBlockTracking(config, configSpec, updateArgs.ConfigSpec, vmCtx.VM.Spec) UpdateConfigSpecFirmware(config, configSpec, vmCtx.VM) - UpdateConfigSpecDeviceGroups(config, configSpec, updateArgs.ConfigSpec) return configSpec } diff --git a/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go b/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go index bd3098660..13f8209b6 100644 --- a/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go +++ b/pkg/vmprovider/providers/vsphere2/session/session_vm_update_test.go @@ -647,59 +647,6 @@ var _ = Describe("Update ConfigSpec", func() { }) }) - Context("DeviceGroups", func() { - var classConfigSpec *vimTypes.VirtualMachineConfigSpec - - BeforeEach(func() { - classConfigSpec = &vimTypes.VirtualMachineConfigSpec{} - }) - - It("No DeviceGroups set in class config spec", func() { - session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) - Expect(configSpec.DeviceGroups).To(BeNil()) - }) - - It("DeviceGroups set in class config spec", func() { - classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ - DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ - &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ - GroupInstanceKey: int32(400), - }, - }, - } - - session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) - Expect(configSpec.DeviceGroups).NotTo(BeNil()) - Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) - deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() - Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) - }) - - It("configInfo DeviceGroups set with vals different than the class config spec", func() { - classConfigSpec.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ - DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ - &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ - GroupInstanceKey: int32(400), - }, - }, - } - - config.DeviceGroups = &vimTypes.VirtualMachineVirtualDeviceGroups{ - DeviceGroup: []vimTypes.BaseVirtualMachineVirtualDeviceGroupsDeviceGroup{ - &vimTypes.VirtualMachineVirtualDeviceGroupsDeviceGroup{ - GroupInstanceKey: int32(500), - }, - }, - } - - session.UpdateConfigSpecDeviceGroups(config, configSpec, classConfigSpec) - Expect(configSpec.DeviceGroups).NotTo(BeNil()) - Expect(configSpec.DeviceGroups.DeviceGroup).To(HaveLen(1)) - deviceGroup := configSpec.DeviceGroups.DeviceGroup[0].GetVirtualMachineVirtualDeviceGroupsDeviceGroup() - Expect(deviceGroup.GroupInstanceKey).To(Equal(int32(400))) - }) - }) - Context("Ethernet Card Changes", func() { var expectedList object.VirtualDeviceList var currentList object.VirtualDeviceList From 1853fb561b1a50e513761db427c5cdbc956a7762 Mon Sep 17 00:00:00 2001 From: Sagar Muchhal Date: Mon, 22 May 2023 14:05:34 -0700 Subject: [PATCH 53/54] Adds minimum h/w version to VirtualMachine CRD Introduces a new field on the VirtualMachine CRD which allows the user to define the minimum hardware version for the provisioned VM. The behavior of the field is as follows: - If the VMClassAsConfig FSS is enabled, it will override the hardware version derived from the VMClass's ConfigSpec if it is lower than the minimum version field. Otherwise, the value from the ConfigSpec is honored. - If the VMClassAsConfig FSS is disabled, it will reconfigure the VM post creation before the power on operation to ensure that the h/w version is atleast set to the minimum version. This means, it can override the hardware version of the OVA with a hardware version value lower than this field. Signed-off-by: Sagar Muchhal --- api/v1alpha1/virtualmachine_types.go | 47 +++++++++++++ api/v1alpha2/virtualmachine_types.go | 49 ++++++++++++- ...vmoperator.vmware.com_virtualmachines.yaml | 68 ++++++++++++++++++- pkg/util/configspec.go | 18 +++++ pkg/util/configspec_test.go | 39 +++++++++++ pkg/util/hardware_version.go | 29 ++++++++ pkg/util/hardware_version_test.go | 28 ++++++++ .../contentlibrary/content_library_utils.go | 25 +------ .../content_library_utils_test.go | 18 ----- .../vsphere/session/session_vm_status.go | 2 + .../providers/vsphere/vmprovider_vm.go | 5 +- .../providers/vsphere/vmprovider_vm_test.go | 34 ++++++++++ .../vsphere2/vmlifecycle/update_status.go | 2 + .../vmlifecycle/update_status_test.go | 21 ++++++ .../providers/vsphere2/vmprovider_vm.go | 2 + .../providers/vsphere2/vmprovider_vm_test.go | 30 ++++++++ test/builder/util.go | 12 ++-- test/builder/utila2.go | 11 +-- .../validation/virtualmachine_validator.go | 2 + .../virtualmachine_validator_intg_test.go | 12 ++++ .../validation/virtualmachine_validator.go | 4 +- .../virtualmachine_validator_intg_test.go | 12 ++++ 22 files changed, 414 insertions(+), 56 deletions(-) create mode 100644 pkg/util/hardware_version.go create mode 100644 pkg/util/hardware_version_test.go diff --git a/api/v1alpha1/virtualmachine_types.go b/api/v1alpha1/virtualmachine_types.go index b562f50e7..3d20e64e7 100644 --- a/api/v1alpha1/virtualmachine_types.go +++ b/api/v1alpha1/virtualmachine_types.go @@ -486,6 +486,44 @@ type VirtualMachineSpec struct { // AdvancedOptions describes a set of optional, advanced options for configuring a VirtualMachine AdvancedOptions *VirtualMachineAdvancedOptions `json:"advancedOptions,omitempty"` + + // MinHardwareVersion specifies the desired minimum hardware version + // for this VM. + // + // Usually the VM's hardware version is derived from: + // 1. the VirtualMachineClass used to deploy the VM provided by the ClassName field + // 2. the datacenter/cluster/host default hardware version + // Setting this field will ensure that the hardware version of the VM + // is at least set to the specified value. To enforce this, it will override + // the value from the VirtualMachineClass. + // + // This field is never updated to reflect the derived hardware version. + // Instead, VirtualMachineStatus.HardwareVersion surfaces + // the observed hardware version. + // + // Please note, setting this field's value to N ensures a VM's hardware + // version is equal to or greater than N. For example, if a VM's observed + // hardware version is 10 and this field's value is 13, then the VM will be + // upgraded to hardware version 13. However, if the observed hardware + // version is 17 and this field's value is 13, no change will occur. + // + // Several features are hardware version dependent, for example: + // + // * NVMe Controllers >= 14 + // * Dynamic Direct Path I/O devices >= 17 + // + // Please refer to https://kb.vmware.com/s/article/1003746 for a list of VM + // hardware versions. + // + // It is important to remember that a VM's hardware version may not be + // downgraded and upgrading a VM deployed from an image based on an older + // hardware version to a more recent one may result in unpredictable + // behavior. In other words, please be careful when choosing to upgrade a + // VM to a newer hardware version. + // + // +optional + // +kubebuilder:validation:Minimum=13 + MinHardwareVersion int32 `json:"minHardwareVersion,omitempty"` } // VirtualMachineAdvancedOptions describes a set of optional, advanced options for configuring a VirtualMachine. @@ -602,6 +640,15 @@ type VirtualMachineStatus struct { // LastRestartTime describes the last time the VM was restarted. // +optional LastRestartTime *metav1.Time `json:"lastRestartTime,omitempty"` + + // HardwareVersion describes the VirtualMachine resource's observed + // hardware version. + // + // Please refer to VirtualMachineSpec.MinHardwareVersion for more + // information on the topic of a VM's hardware version. + // + // +optional + HardwareVersion int32 `json:"hardwareVersion,omitempty"` } func (vm *VirtualMachine) GetConditions() Conditions { diff --git a/api/v1alpha2/virtualmachine_types.go b/api/v1alpha2/virtualmachine_types.go index 843253780..b1546dafe 100644 --- a/api/v1alpha2/virtualmachine_types.go +++ b/api/v1alpha2/virtualmachine_types.go @@ -174,7 +174,7 @@ type VirtualMachineSpec struct { // +optional ImageName string `json:"imageName,omitempty"` - // Class describes the name of the VirtualMachineClass resource used to + // ClassName describes the name of the VirtualMachineClass resource used to // deploy this VM. // // This field is optional in the cases where there exists a sensible @@ -335,6 +335,44 @@ type VirtualMachineSpec struct { // // +optional Reserved VirtualMachineReservedSpec `json:"reserved,omitempty"` + + // MinHardwareVersion specifies the desired minimum hardware version + // for this VM. + // + // Usually the VM's hardware version is derived from: + // 1. the VirtualMachineClass used to deploy the VM provided by the ClassName field + // 2. the datacenter/cluster/host default hardware version + // Setting this field will ensure that the hardware version of the VM + // is at least set to the specified value. To enforce this, it will override + // the value from the VirtualMachineClass. + // + // This field is never updated to reflect the derived hardware version. + // Instead, VirtualMachineStatus.HardwareVersion surfaces + // the observed hardware version. + // + // Please note, setting this field's value to N ensures a VM's hardware + // version is equal to or greater than N. For example, if a VM's observed + // hardware version is 10 and this field's value is 13, then the VM will be + // upgraded to hardware version 13. However, if the observed hardware + // version is 17 and this field's value is 13, no change will occur. + // + // Several features are hardware version dependent, for example: + // + // * NVMe Controllers >= 14 + // * Dynamic Direct Path I/O devices >= 17 + // + // Please refer to https://kb.vmware.com/s/article/1003746 for a list of VM + // hardware versions. + // + // It is important to remember that a VM's hardware version may not be + // downgraded and upgrading a VM deployed from an image based on an older + // hardware version to a more recent one may result in unpredictable + // behavior. In other words, please be careful when choosing to upgrade a + // VM to a newer hardware version. + // + // +optional + // +kubebuilder:validation:Minimum=13 + MinHardwareVersion int32 `json:"minHardwareVersion,omitempty"` } // VirtualMachineReservedSpec describes a set of VM configuration options @@ -455,6 +493,15 @@ type VirtualMachineStatus struct { // // +optional LastRestartTime *metav1.Time `json:"lastRestartTime,omitempty"` + + // HardwareVersion describes the VirtualMachine resource's observed + // hardware version. + // + // Please refer to VirtualMachineSpec.MinHardwareVersion for more + // information on the topic of a VM's hardware version. + // + // +optional + HardwareVersion int32 `json:"hardwareVersion,omitempty"` } // +kubebuilder:object:root=true diff --git a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml index 12a4f4341..5cf3895a5 100644 --- a/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml +++ b/config/crd/bases/vmoperator.vmware.com_virtualmachines.yaml @@ -104,6 +104,33 @@ spec: be introspected to discover identifying attributes that may help users to identify the desired image to use. type: string + minHardwareVersion: + description: "MinHardwareVersion specifies the desired minimum hardware + version for this VM. \n Usually the VM's hardware version is derived + from: 1. the VirtualMachineClass used to deploy the VM provided + by the ClassName field 2. the datacenter/cluster/host default hardware + version Setting this field will ensure that the hardware version + of the VM is at least set to the specified value. To enforce this, + it will override the value from the VirtualMachineClass. \n This + field is never updated to reflect the derived hardware version. + Instead, VirtualMachineStatus.HardwareVersion surfaces the observed + hardware version. \n Please note, setting this field's value to + N ensures a VM's hardware version is equal to or greater than N. + For example, if a VM's observed hardware version is 10 and this + field's value is 13, then the VM will be upgraded to hardware version + 13. However, if the observed hardware version is 17 and this field's + value is 13, no change will occur. \n Several features are hardware + version dependent, for example: \n * NVMe Controllers \t\t + >= 14 * Dynamic Direct Path I/O devices >= 17 \n Please refer to + https://kb.vmware.com/s/article/1003746 for a list of VM hardware + versions. \n It is important to remember that a VM's hardware version + may not be downgraded and upgrading a VM deployed from an image + based on an older hardware version to a more recent one may result + in unpredictable behavior. In other words, please be careful when + choosing to upgrade a VM to a newer hardware version." + format: int32 + minimum: 13 + type: integer networkInterfaces: description: NetworkInterfaces describes a list of VirtualMachineNetworkInterfaces to be configured on the VirtualMachine instance. Each of these VirtualMachineNetworkInterfaces @@ -494,6 +521,12 @@ spec: - type type: object type: array + hardwareVersion: + description: "HardwareVersion describes the VirtualMachine resource's + observed hardware version. \n Please refer to VirtualMachineSpec.MinHardwareVersion + for more information on the topic of a VM's hardware version." + format: int32 + type: integer host: description: Host describes the hostname or IP address of the infrastructure host that the VirtualMachine is executing on. @@ -1500,7 +1533,7 @@ spec: type: object type: object className: - description: "Class describes the name of the VirtualMachineClass + description: "ClassName describes the name of the VirtualMachineClass resource used to deploy this VM. \n This field is optional in the cases where there exists a sensible default value, such as when there is a single VirtualMachineClass resource available in the @@ -1518,6 +1551,33 @@ spec: default value, such as when there is a single VirtualMachineImage resource available in the same Namespace as the VM being deployed." type: string + minHardwareVersion: + description: "MinHardwareVersion specifies the desired minimum hardware + version for this VM. \n Usually the VM's hardware version is derived + from: 1. the VirtualMachineClass used to deploy the VM provided + by the ClassName field 2. the datacenter/cluster/host default hardware + version Setting this field will ensure that the hardware version + of the VM is at least set to the specified value. To enforce this, + it will override the value from the VirtualMachineClass. \n This + field is never updated to reflect the derived hardware version. + Instead, VirtualMachineStatus.HardwareVersion surfaces the observed + hardware version. \n Please note, setting this field's value to + N ensures a VM's hardware version is equal to or greater than N. + For example, if a VM's observed hardware version is 10 and this + field's value is 13, then the VM will be upgraded to hardware version + 13. However, if the observed hardware version is 17 and this field's + value is 13, no change will occur. \n Several features are hardware + version dependent, for example: \n * NVMe Controllers \t\t + >= 14 * Dynamic Direct Path I/O devices >= 17 \n Please refer to + https://kb.vmware.com/s/article/1003746 for a list of VM hardware + versions. \n It is important to remember that a VM's hardware version + may not be downgraded and upgrading a VM deployed from an image + based on an older hardware version to a more recent one may result + in unpredictable behavior. In other words, please be careful when + choosing to upgrade a VM to a newer hardware version." + format: int32 + minimum: 13 + type: integer network: description: "Network describes the desired network configuration for the VM. \n Please note this value may be omitted entirely and @@ -2203,6 +2263,12 @@ spec: - type type: object type: array + hardwareVersion: + description: "HardwareVersion describes the VirtualMachine resource's + observed hardware version. \n Please refer to VirtualMachineSpec.MinHardwareVersion + for more information on the topic of a VM's hardware version." + format: int32 + type: integer host: description: Host describes the hostname or IP address of the infrastructure host where the VM is executed. diff --git a/pkg/util/configspec.go b/pkg/util/configspec.go index 8a62d364b..041f65cd6 100644 --- a/pkg/util/configspec.go +++ b/pkg/util/configspec.go @@ -5,6 +5,7 @@ package util import ( "bytes" + "fmt" "reflect" "github.com/vmware/govmomi/vim25" @@ -215,3 +216,20 @@ func MergeExtraConfig(extraConfig []vimTypes.BaseOptionValue, newMap map[string] } return merged } + +// EnsureMinHardwareVersionInConfigSpec ensures that the hardware version in the ConfigSpec +// is at least equal to the passed minimum hardware version value. +func EnsureMinHardwareVersionInConfigSpec(configSpec *vimTypes.VirtualMachineConfigSpec, minVersion int32) { + if minVersion == 0 { + return + } + + configSpecHwVersion := int32(0) + if configSpec.Version != "" { + configSpecHwVersion = ParseVirtualHardwareVersion(configSpec.Version) + } + if minVersion > configSpecHwVersion { + configSpecHwVersion = minVersion + } + configSpec.Version = fmt.Sprintf("vmx-%d", configSpecHwVersion) +} diff --git a/pkg/util/configspec_test.go b/pkg/util/configspec_test.go index 9d895f53c..bc631bf7b 100644 --- a/pkg/util/configspec_test.go +++ b/pkg/util/configspec_test.go @@ -165,6 +165,45 @@ var _ = Describe("ConfigSpec Util", func() { Expect(cmp.Diff(cs2, cs3)).To(BeEmpty()) }) }) + + Context("EnsureMinHardwareVersionInConfigSpec", func() { + When("minimum hardware version is unset", func() { + It("does not change the existing value of the configSpec's version", func() { + configSpec := &vimTypes.VirtualMachineConfigSpec{Version: "vmx-15"} + util.EnsureMinHardwareVersionInConfigSpec(configSpec, 0) + + Expect(configSpec.Version).To(Equal("vmx-15")) + }) + + It("does not set the configSpec's version", func() { + configSpec := &vimTypes.VirtualMachineConfigSpec{} + util.EnsureMinHardwareVersionInConfigSpec(configSpec, 0) + + Expect(configSpec.Version).To(BeEmpty()) + }) + }) + + It("overrides the hardware version if the existing version is lesser", func() { + configSpec := &vimTypes.VirtualMachineConfigSpec{Version: "vmx-15"} + util.EnsureMinHardwareVersionInConfigSpec(configSpec, 17) + + Expect(configSpec.Version).To(Equal("vmx-17")) + }) + + It("sets the hardware version if the existing version is unset", func() { + configSpec := &vimTypes.VirtualMachineConfigSpec{} + util.EnsureMinHardwareVersionInConfigSpec(configSpec, 16) + + Expect(configSpec.Version).To(Equal("vmx-16")) + }) + + It("overrides the hardware version if the existing version is set incorrectly", func() { + configSpec := &vimTypes.VirtualMachineConfigSpec{Version: "foo"} + util.EnsureMinHardwareVersionInConfigSpec(configSpec, 17) + + Expect(configSpec.Version).To(Equal("vmx-17")) + }) + }) }) var _ = Describe("RemoveDevicesFromConfigSpec", func() { diff --git a/pkg/util/hardware_version.go b/pkg/util/hardware_version.go new file mode 100644 index 000000000..416d5a2f3 --- /dev/null +++ b/pkg/util/hardware_version.go @@ -0,0 +1,29 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package util + +import ( + "regexp" + "strconv" +) + +var vmxRe = regexp.MustCompile(`vmx-(\d+)`) + +// ParseVirtualHardwareVersion parses the virtual hardware version +// For eg. "vmx-15" returns 15. +func ParseVirtualHardwareVersion(vmxVersion string) int32 { + // obj matches the full string and the submatch (\d+) + // and return a []string with values + obj := vmxRe.FindStringSubmatch(vmxVersion) + if len(obj) != 2 { + return 0 + } + + version, err := strconv.ParseInt(obj[1], 10, 32) + if err != nil { + return 0 + } + + return int32(version) +} diff --git a/pkg/util/hardware_version_test.go b/pkg/util/hardware_version_test.go new file mode 100644 index 000000000..b9b8a2e54 --- /dev/null +++ b/pkg/util/hardware_version_test.go @@ -0,0 +1,28 @@ +// Copyright (c) 2023 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package util_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vmware-tanzu/vm-operator/pkg/util" +) + +var _ = Describe("ParseVirtualHardwareVersion", func() { + It("empty hardware string", func() { + vmxHwVersionString := "" + Expect(util.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) + }) + + It("invalid hardware string", func() { + vmxHwVersionString := "blah" + Expect(util.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) + }) + + It("valid hardware version string eg. vmx-15", func() { + vmxHwVersionString := "vmx-15" + Expect(util.ParseVirtualHardwareVersion(vmxHwVersionString)).To(Equal(int32(15))) + }) +}) diff --git a/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_utils.go b/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_utils.go index e9ef8e03d..84b3f1d16 100644 --- a/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_utils.go +++ b/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_utils.go @@ -8,8 +8,6 @@ import ( "fmt" "io" "net/url" - "regexp" - "strconv" "strings" "github.com/vmware/govmomi/ovf" @@ -24,29 +22,10 @@ import ( vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" "github.com/vmware-tanzu/vm-operator/pkg/conditions" "github.com/vmware-tanzu/vm-operator/pkg/lib" + "github.com/vmware-tanzu/vm-operator/pkg/util" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" ) -var vmxRe = regexp.MustCompile(`vmx-(\d+)`) - -// ParseVirtualHardwareVersion parses the virtual hardware version -// For eg. "vmx-15" returns 15. -func ParseVirtualHardwareVersion(vmxVersion string) int32 { - // obj matches the full string and the submatch (\d+) - // and return a []string with values - obj := vmxRe.FindStringSubmatch(vmxVersion) - if len(obj) != 2 { - return 0 - } - - version, err := strconv.ParseInt(obj[1], 10, 32) - if err != nil { - return 0 - } - - return int32(version) -} - // LibItemToVirtualMachineImage converts a given library item and its attributes to return a // VirtualMachineImage that represents a k8s-native view of the item. func LibItemToVirtualMachineImage( @@ -170,7 +149,7 @@ func updateImageSpecWithOvfVirtualSystem(imageSpec *vmopv1.VirtualMachineImageSp if virtualHwSection := ovfVirtualSystem.VirtualHardware; len(virtualHwSection) > 0 { hw := virtualHwSection[0] if hw.System != nil && hw.System.VirtualSystemType != nil { - hwVersion = ParseVirtualHardwareVersion(*hw.System.VirtualSystemType) + hwVersion = util.ParseVirtualHardwareVersion(*hw.System.VirtualSystemType) } } diff --git a/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_utils_test.go b/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_utils_test.go index 802a70f43..33f3a95a3 100644 --- a/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_utils_test.go +++ b/pkg/vmprovider/providers/vsphere/contentlibrary/content_library_utils_test.go @@ -16,30 +16,12 @@ import ( "k8s.io/utils/pointer" vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" - "github.com/vmware-tanzu/vm-operator/pkg/conditions" "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/contentlibrary" ) -var _ = Describe("ParseVirtualHardwareVersion", func() { - It("empty hardware string", func() { - vmxHwVersionString := "" - Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) - }) - - It("invalid hardware string", func() { - vmxHwVersionString := "blah" - Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(BeZero()) - }) - - It("valid hardware version string eg. vmx-15", func() { - vmxHwVersionString := "vmx-15" - Expect(contentlibrary.ParseVirtualHardwareVersion(vmxHwVersionString)).To(Equal(int32(15))) - }) -}) - var _ = Describe("LibItemToVirtualMachineImage", func() { const ( versionKey = "vmware-system-version" diff --git a/pkg/vmprovider/providers/vsphere/session/session_vm_status.go b/pkg/vmprovider/providers/vsphere/session/session_vm_status.go index 23518b513..7bcdf7b98 100644 --- a/pkg/vmprovider/providers/vsphere/session/session_vm_status.go +++ b/pkg/vmprovider/providers/vsphere/session/session_vm_status.go @@ -16,6 +16,7 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/util" res "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/resources" ) @@ -108,6 +109,7 @@ func (s *Session) updateVMStatus( vm.Status.UniqueID = resVM.MoRef().Value vm.Status.BiosUUID = summary.Config.Uuid vm.Status.InstanceUUID = summary.Config.InstanceUuid + vm.Status.HardwareVersion = util.ParseVirtualHardwareVersion(summary.Config.HwVersion) if host := summary.Runtime.Host; host != nil { hostSystem := object.NewHostSystem(s.Client.VimClient(), *host) diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm.go index a1a4f9bf3..84b743390 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm.go @@ -26,7 +26,6 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/util" vcclient "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/client" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/constants" - "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/contentlibrary" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/instancestorage" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/network" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere/placement" @@ -214,7 +213,7 @@ func (vs *vSphereVMProvider) GetVirtualMachineHardwareVersion( return 0, err } - return contentlibrary.ParseVirtualHardwareVersion(o.Config.Version), nil + return util.ParseVirtualHardwareVersion(o.Config.Version), nil } func (vs *vSphereVMProvider) createVirtualMachine( @@ -675,6 +674,8 @@ func (vs *vSphereVMProvider) vmCreateGenConfigSpec( createArgs.ConfigSpec.Version = fmt.Sprintf("vmx-%d", version) } } + + util.EnsureMinHardwareVersionInConfigSpec(createArgs.ConfigSpec, vmCtx.VM.Spec.MinHardwareVersion) } func (vs *vSphereVMProvider) vmCreateValidateArgs( diff --git a/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go index 0c3d3aa2b..32884a003 100644 --- a/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go +++ b/pkg/vmprovider/providers/vsphere/vmprovider_vm_test.go @@ -854,6 +854,7 @@ func vmTests() { Context("VM spec has a PVC", func() { BeforeEach(func() { + vm.Spec.MinHardwareVersion = 14 vm.Spec.Volumes = []vmopv1.VirtualMachineVolume{ { Name: "dummy-vol", @@ -876,11 +877,44 @@ func vmTests() { It("creates a VM with a hardware version minimum supported for PVCs", func() { var o mo.VirtualMachine Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + // The min version required for PVCs is honored even when the min h/w + // version is set to a different value. + Expect(o.Config.Version).NotTo(Equal("vmx-14")) Expect(o.Config.Version).To(Equal(fmt.Sprintf("vmx-%d", constants.MinSupportedHWVersionForPVC))) }) }) }) + Context("VM Class Config specifies a hardware version", func() { + BeforeEach(func() { + configSpec = &types.VirtualMachineConfigSpec{Version: "vmx-14"} + }) + + When("The minimum hardware version on the VMSpec is greater than VMClass", func() { + BeforeEach(func() { + vm.Spec.MinHardwareVersion = 15 + }) + + It("updates the VM to minimum hardware version from the Spec", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal("vmx-15")) + }) + }) + + When("The minimum hardware version on the VMSpec is less than VMClass", func() { + BeforeEach(func() { + vm.Spec.MinHardwareVersion = 13 + }) + + It("uses the hardware version from the VMClass", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal("vmx-14")) + }) + }) + }) + Context("VMClassAsConfig FSS is Enabled", func() { BeforeEach(func() { diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go index 21dc3c7c4..330128201 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status.go @@ -21,6 +21,7 @@ import ( "github.com/vmware-tanzu/vm-operator/pkg/context" "github.com/vmware-tanzu/vm-operator/pkg/lib" "github.com/vmware-tanzu/vm-operator/pkg/topology" + "github.com/vmware-tanzu/vm-operator/pkg/util" "github.com/vmware-tanzu/vm-operator/pkg/vmprovider/providers/vsphere2/virtualmachine" ) @@ -74,6 +75,7 @@ func UpdateStatus( vm.Status.BiosUUID = summary.Config.Uuid vm.Status.InstanceUUID = summary.Config.InstanceUuid vm.Status.Network = getGuestNetworkStatus(vmMO.Guest) + vm.Status.HardwareVersion = util.ParseVirtualHardwareVersion(summary.Config.HwVersion) vm.Status.Host, err = getRuntimeHostHostname(vmCtx, vcVM, summary.Runtime.Host) if err != nil { diff --git a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go index f9050ea97..900768f97 100644 --- a/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmlifecycle/update_status_test.go @@ -126,6 +126,27 @@ var _ = Describe("UpdateStatus", func() { }) }) }) + + Context("Copies values to the VM status", func() { + biosUUID, instanceUUID := "f7c371d6-2003-5a48-9859-3bc9a8b0890", "6132d223-1566-5921-bc3b-df91ece09a4d" + BeforeEach(func() { + vmMO.Summary = types.VirtualMachineSummary{ + Config: types.VirtualMachineConfigSummary{ + Uuid: biosUUID, + InstanceUuid: instanceUUID, + HwVersion: "vmx-19", + }, + } + }) + + It("sets the summary config values in the status", func() { + status := vmCtx.VM.Status + Expect(status).NotTo(BeNil()) + Expect(status.BiosUUID).To(Equal(biosUUID)) + Expect(status.InstanceUUID).To(Equal(instanceUUID)) + Expect(status.HardwareVersion).To(Equal(int32(19))) + }) + }) }) var _ = Describe("VirtualMachineTools Status to VM Status Condition", func() { diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go index c3046d060..41de29915 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm.go @@ -899,6 +899,8 @@ func (vs *vSphereVMProvider) vmCreateGenConfigSpec( } } + util.EnsureMinHardwareVersionInConfigSpec(createArgs.ConfigSpec, vmCtx.VM.Spec.MinHardwareVersion) + err := vs.vmCreateGenConfigSpecExtraConfig(vmCtx, createArgs) if err != nil { return err diff --git a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go index cfb000f8d..692bd4dcd 100644 --- a/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go +++ b/pkg/vmprovider/providers/vsphere2/vmprovider_vm_test.go @@ -852,6 +852,36 @@ func vmTests() { }) }) + Context("VM Class Config specifies a hardware version", func() { + BeforeEach(func() { + configSpec = &types.VirtualMachineConfigSpec{Version: "vmx-14"} + }) + + When("The minimum hardware version on the VMSpec is greater than VMClass", func() { + BeforeEach(func() { + vm.Spec.MinHardwareVersion = 15 + }) + + It("updates the VM to minimum hardware version from the Spec", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal("vmx-15")) + }) + }) + + When("The minimum hardware version on the VMSpec is less than VMClass", func() { + BeforeEach(func() { + vm.Spec.MinHardwareVersion = 13 + }) + + It("uses the hardware version from the VMClass", func() { + var o mo.VirtualMachine + Expect(vcVM.Properties(ctx, vcVM.Reference(), nil, &o)).To(Succeed()) + Expect(o.Config.Version).To(Equal("vmx-14")) + }) + }) + }) + Context("VMClassAsConfig FSS is Enabled", func() { BeforeEach(func() { diff --git a/test/builder/util.go b/test/builder/util.go index 196a1124d..60cdcf9c3 100644 --- a/test/builder/util.go +++ b/test/builder/util.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/yaml" imgregv1a1 "github.com/vmware-tanzu/image-registry-operator-api/api/v1alpha1" + vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" "github.com/vmware-tanzu/vm-operator/pkg/lib" @@ -247,11 +248,12 @@ func DummyVirtualMachine() *vmopv1.VirtualMachine { Annotations: map[string]string{}, }, Spec: vmopv1.VirtualMachineSpec{ - ImageName: DummyImageName, - ClassName: DummyClassName, - PowerState: vmopv1.VirtualMachinePoweredOn, - PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, - SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + ImageName: DummyImageName, + ClassName: DummyClassName, + PowerState: vmopv1.VirtualMachinePoweredOn, + PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, + SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + MinHardwareVersion: 13, NetworkInterfaces: []vmopv1.VirtualMachineNetworkInterface{ { NetworkName: DummyNetworkName, diff --git a/test/builder/utila2.go b/test/builder/utila2.go index 141d93cf5..97e9a4739 100644 --- a/test/builder/utila2.go +++ b/test/builder/utila2.go @@ -139,11 +139,12 @@ func DummyVirtualMachineA2() *vmopv1.VirtualMachine { Annotations: map[string]string{}, }, Spec: vmopv1.VirtualMachineSpec{ - ImageName: DummyImageName, - ClassName: DummyClassName, - PowerState: vmopv1.VirtualMachinePowerStateOn, - PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, - SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + ImageName: DummyImageName, + ClassName: DummyClassName, + PowerState: vmopv1.VirtualMachinePowerStateOn, + PowerOffMode: vmopv1.VirtualMachinePowerOpModeHard, + SuspendMode: vmopv1.VirtualMachinePowerOpModeHard, + MinHardwareVersion: 13, Volumes: []vmopv1.VirtualMachineVolume{ { Name: DummyVolumeName, diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go index 775dc01d6..b179281bc 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator.go @@ -146,6 +146,7 @@ func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Resp // - ClassName // - StorageClass // - ResourcePolicyName +// - Minimum VM Hardware Version // Following fields can only be updated when the VM is powered off. // - Ports @@ -715,6 +716,7 @@ func (v validator) validateImmutableFields(ctx *context.WebhookRequestContext, v allErrs = append(allErrs, validation.ValidateImmutableField(vm.Spec.ClassName, oldVM.Spec.ClassName, specPath.Child("className"))...) allErrs = append(allErrs, validation.ValidateImmutableField(vm.Spec.StorageClass, oldVM.Spec.StorageClass, specPath.Child("storageClass"))...) allErrs = append(allErrs, validation.ValidateImmutableField(vm.Spec.ResourcePolicyName, oldVM.Spec.ResourcePolicyName, specPath.Child("resourcePolicyName"))...) + allErrs = append(allErrs, validation.ValidateImmutableField(vm.Spec.MinHardwareVersion, oldVM.Spec.MinHardwareVersion, specPath.Child("minHardwareVersion"))...) return allErrs } diff --git a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_intg_test.go b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_intg_test.go index 9c205cecf..d3c93e1c2 100644 --- a/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_intg_test.go +++ b/webhooks/virtualmachine/v1alpha1/validation/virtualmachine_validator_intg_test.go @@ -163,6 +163,18 @@ func intgTestsValidateUpdate() { }) }) + When("update is performed with changed minimum hardware version", func() { + BeforeEach(func() { + ctx.vm.Spec.MinHardwareVersion += 2 + }) + It("should deny the request", func() { + Expect(err).To(HaveOccurred()) + expectedPath := field.NewPath("spec", "minHardwareVersion") + Expect(err.Error()).To(ContainSubstring(expectedPath.String())) + Expect(err.Error()).To(ContainSubstring(immutableFieldMsg)) + }) + }) + Context("VirtualMachine update while VM is powered on", func() { BeforeEach(func() { ctx.vm.Spec.PowerState = "poweredOn" diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index 59b3a9925..e848b14bc 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -135,6 +135,7 @@ func (v validator) ValidateDelete(*context.WebhookRequestContext) admission.Resp // - ClassName // - StorageClass // - ResourcePolicyName +// - Minimum VM Hardware Version // // Following fields can only be changed when the VM is powered off. // - Bootstrap @@ -738,13 +739,14 @@ func (v validator) validateUpdatesWhenPoweredOn(ctx *context.WebhookRequestConte return allErrs } -func (v validator) validateImmutableFields(ctx *context.WebhookRequestContext, vm, oldVM *vmopv1.VirtualMachine) field.ErrorList { +func (v validator) validateImmutableFields(_ *context.WebhookRequestContext, vm, oldVM *vmopv1.VirtualMachine) field.ErrorList { var allErrs field.ErrorList specPath := field.NewPath("spec") allErrs = append(allErrs, validation.ValidateImmutableField(vm.Spec.ImageName, oldVM.Spec.ImageName, specPath.Child("imageName"))...) allErrs = append(allErrs, validation.ValidateImmutableField(vm.Spec.ClassName, oldVM.Spec.ClassName, specPath.Child("className"))...) allErrs = append(allErrs, validation.ValidateImmutableField(vm.Spec.StorageClass, oldVM.Spec.StorageClass, specPath.Child("storageClass"))...) + allErrs = append(allErrs, validation.ValidateImmutableField(vm.Spec.MinHardwareVersion, oldVM.Spec.MinHardwareVersion, specPath.Child("minHardwareVersion"))...) // TODO: More checks. // TODO: Allow privilege? diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_intg_test.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_intg_test.go index e241ec925..08898c837 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_intg_test.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_intg_test.go @@ -128,6 +128,18 @@ func intgTestsValidateUpdate() { }) }) + When("update is performed with changed minimum hardware version", func() { + BeforeEach(func() { + ctx.vm.Spec.MinHardwareVersion += 2 + }) + It("should deny the request", func() { + Expect(err).To(HaveOccurred()) + expectedPath := field.NewPath("spec", "minHardwareVersion") + Expect(err.Error()).To(ContainSubstring(expectedPath.String())) + Expect(err.Error()).To(ContainSubstring(immutableFieldMsg)) + }) + }) + Context("VirtualMachine update while VM is powered on", func() { BeforeEach(func() { ctx.vm.Spec.PowerState = vmopv1.VirtualMachinePowerStateOn From 387ff3ff4c02288f06da8b68bfd4b01df185aa81 Mon Sep 17 00:00:00 2001 From: Yiyi Zhou <91219164+zyiyi11@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:40:11 -0800 Subject: [PATCH 54/54] Add validation on Bootstrap vAppConfig Properties (#269) --- .../validation/virtualmachine_validator.go | 18 +++- .../virtualmachine_validator_unit_test.go | 83 ++++++++++++++++--- 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go index 857126444..b98f235eb 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator.go @@ -203,7 +203,7 @@ func (v validator) validateBootstrap( hasCloudConfig := !equality.Semantic.DeepEqual(cloudInit.CloudConfig, cloudinit.CloudConfig{}) hasRawCloudConfig := !equality.Semantic.DeepEqual(cloudInit.RawCloudConfig, corev1.SecretKeySelector{}) if hasCloudConfig && hasRawCloudConfig { - allErrs = append(allErrs, field.Invalid(p, cloudInit, + allErrs = append(allErrs, field.Invalid(p, "cloudInit", "cloudConfig and rawCloudConfig are mutually exclusive")) } } @@ -229,7 +229,7 @@ func (v validator) validateBootstrap( hasSysPrep := !equality.Semantic.DeepEqual(sysPrep.Sysprep, sysprep.Sysprep{}) hasRawSysPrep := !equality.Semantic.DeepEqual(sysPrep.RawSysprep, corev1.SecretKeySelector{}) if hasSysPrep && hasRawSysPrep { - allErrs = append(allErrs, field.Invalid(p, sysPrep, + allErrs = append(allErrs, field.Invalid(p, "sysPrep", "sysprep and rawSysprep are mutually exclusive")) } } else { @@ -246,9 +246,21 @@ func (v validator) validateBootstrap( } if len(vAppConfig.Properties) != 0 && len(vAppConfig.RawProperties) != 0 { - allErrs = append(allErrs, field.TypeInvalid(p, vAppConfig, + allErrs = append(allErrs, field.TypeInvalid(p, "vAppConfig", "properties and rawProperties are mutually exclusive")) } + + for _, property := range vAppConfig.Properties { + if key := property.Key; key == "" { + allErrs = append(allErrs, field.Invalid(p.Child("properties").Child("key"), "key", + "key is a required field in vAppConfig Properties")) + } + if value := property.Value; value.From != nil && value.Value != nil { + allErrs = append(allErrs, field.Invalid(p.Child("properties").Child("value"), "value", + "from and value is mutually exclusive")) + } + } + } return allErrs diff --git a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go index e7f6aa704..18dc12240 100644 --- a/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go +++ b/webhooks/virtualmachine/v1alpha2/validation/virtualmachine_validator_unit_test.go @@ -362,10 +362,12 @@ func unitTestsValidateCreate() { doValidateWithMsg := func(msgs ...string) func(admission.Response) { return func(response admission.Response) { - reasons := string(response.Result.Reason) + reasons := strings.Split(string(response.Result.Reason), ", ") for _, m := range msgs { - Expect(reasons).To(ContainSubstring(m)) + Expect(reasons).To(ContainElement(m)) } + // This may be overly strict in some cases but catches missed assertions. + Expect(reasons).To(HaveLen(len(msgs))) } } @@ -420,17 +422,22 @@ func unitTestsValidateCreate() { ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} ctx.vm.Spec.Bootstrap.LinuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} }, - validate: doValidateWithMsg("CloudInit may not be used with any other bootstrap provider", - "LinuxPrep may not be used with either CloudInit or Sysprep bootstrap providers"), + validate: doValidateWithMsg( + `spec.bootstrap.cloudInit: Forbidden: CloudInit may not be used with any other bootstrap provider`, + `spec.bootstrap.linuxPrep: Forbidden: LinuxPrep may not be used with either CloudInit or Sysprep bootstrap providers`), }, ), Entry("disallow CloudInit and Sysprep specified at the same time", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + Expect(os.Setenv(lib.WindowsSysprepFSS, "true")).To(Succeed()) ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} }, - validate: doValidateWithMsg("CloudInit may not be used with any other bootstrap provider"), + validate: doValidateWithMsg( + `spec.bootstrap.cloudInit: Forbidden: CloudInit may not be used with any other bootstrap provider`, + `spec.bootstrap.sysprep: Forbidden: Sysprep may not be used with either CloudInit or LinuxPrep bootstrap providers`, + ), }, ), Entry("disallow CloudInit and vAppConfig specified at the same time", @@ -439,16 +446,23 @@ func unitTestsValidateCreate() { ctx.vm.Spec.Bootstrap.CloudInit = &vmopv1.VirtualMachineBootstrapCloudInitSpec{} ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{} }, - validate: doValidateWithMsg("CloudInit may not be used with any other bootstrap provider"), + validate: doValidateWithMsg( + `spec.bootstrap.cloudInit: Forbidden: CloudInit may not be used with any other bootstrap provider`, + `spec.bootstrap.vAppConfig: Forbidden: vAppConfig may not be used in conjunction with CloudInit bootstrap provider`, + ), }, ), Entry("disallow LinuxPrep and Sysprep specified at the same time", testParams{ setup: func(ctx *unitValidatingWebhookContext) { + Expect(os.Setenv(lib.WindowsSysprepFSS, "true")).To(Succeed()) ctx.vm.Spec.Bootstrap.LinuxPrep = &vmopv1.VirtualMachineBootstrapLinuxPrepSpec{} ctx.vm.Spec.Bootstrap.Sysprep = &vmopv1.VirtualMachineBootstrapSysprepSpec{} }, - validate: doValidateWithMsg("LinuxPrep may not be used with either CloudInit or Sysprep bootstrap providers"), + validate: doValidateWithMsg( + `spec.bootstrap.linuxPrep: Forbidden: LinuxPrep may not be used with either CloudInit or Sysprep bootstrap providers`, + `spec.bootstrap.sysprep: Forbidden: Sysprep may not be used with either CloudInit or LinuxPrep bootstrap providers`, + ), }, ), Entry("allow LinuxPrep and vAppConfig specified at the same time", @@ -482,7 +496,9 @@ func unitTestsValidateCreate() { }, } }, - validate: doValidateWithMsg("cloudConfig and rawCloudConfig are mutually exclusive"), + validate: doValidateWithMsg( + `spec.bootstrap.cloudInit: Invalid value: "cloudInit": cloudConfig and rawCloudConfig are mutually exclusive`, + ), }, ), Entry("disallow Sysprep mixing inline Sysprep and RawSysprep when FSS is enabled", @@ -493,7 +509,9 @@ func unitTestsValidateCreate() { ctx.vm.Spec.Bootstrap.Sysprep.Sysprep.GUIRunOnce.Commands = []string{"hello"} ctx.vm.Spec.Bootstrap.Sysprep.RawSysprep.Key = "sysprep-key" }, - validate: doValidateWithMsg("sysprep and rawSysprep are mutually exclusive"), + validate: doValidateWithMsg( + `spec.bootstrap.sysprep: Invalid value: "sysPrep": sysprep and rawSysprep are mutually exclusive`, + ), }, ), Entry("disallow vAppConfig mixing inline Properties and RawProperties", @@ -508,7 +526,52 @@ func unitTestsValidateCreate() { RawProperties: "some-vapp-prop", } }, - validate: doValidateWithMsg("properties and rawProperties are mutually exclusive"), + validate: doValidateWithMsg( + `spec.bootstrap.vAppConfig: Invalid value: "vAppConfig": properties and rawProperties are mutually exclusive`, + ), + }, + ), + + Entry("disallow vAppConfig mixing Properties Value From Secret and direct String pointer", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Key: "key", + Value: common.ValueOrSecretKeySelector{ + From: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "secret-name"}, + Key: "key", + }, + Value: pointer.String("value"), + }, + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.bootstrap.vAppConfig.properties.value: Invalid value: "value": from and value is mutually exclusive`, + ), + }, + ), + + Entry("disallow vAppConfig inline Properties missing Key", + testParams{ + setup: func(ctx *unitValidatingWebhookContext) { + ctx.vm.Spec.Bootstrap.VAppConfig = &vmopv1.VirtualMachineBootstrapVAppConfigSpec{ + Properties: []common.KeyValueOrSecretKeySelectorPair{ + { + Value: common.ValueOrSecretKeySelector{ + Value: pointer.String("value"), + }, + }, + }, + } + }, + validate: doValidateWithMsg( + `spec.bootstrap.vAppConfig.properties.key: Invalid value: "key": key is a required field in vAppConfig Properties`, + ), }, ), )