Merge branch 'main' into vapp-template-lease

Signed-off-by: Giuseppe Maxia <[email protected]>

.changes/v3.11.0/1123-improvements.md

@@ -0,0 +1 @@
+* Add property `upload_any_file` to resource `vcd_catalog_media` to allow uploading any file as catalog media item [GH-1123]

.changes/v3.11.0/1124-improvements.md

@@ -0,0 +1 @@
+* Add property `download_to_file` to data source `vcd_catalog_media` to allow downloading a catalog media item into a file [GH-1124]

.changes/v3.11.0/1140-bug-fixes.md

@@ -0,0 +1,2 @@
+* Fix organization retrieval in `vcd_resource_list` when users fill the `"parent"` field instead of `"org"` [GH-1140]
+* Fix organization retrieval in `vcd_resource_list` when field `"org"` from the provider block was not used [GH-1140]

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.27.0
 	github.com/kr/pretty v0.2.1
-	github.com/vmware/go-vcloud-director/v2 v2.22.0-alpha.6
+	github.com/vmware/go-vcloud-director/v2 v2.22.0-alpha.7
 )
 
 require (
@@ -66,6 +66,6 @@ require (
 	google.golang.org/protobuf v1.30.0 // indirect
 )
 
-replace github.com/vmware/go-vcloud-director/v2 => github.com/dataclouder/go-vcloud-director/v2 v2.17.0-alpha.3.0.20231017090632-1876e0d9beea
+replace github.com/vmware/go-vcloud-director/v2 => github.com/dataclouder/go-vcloud-director/v2 v2.17.0-alpha.3.0.20231023153621-323907328698
 
 // replace github.com/vmware/go-vcloud-director/v2 => ../go-vcloud-director

go.sum

@@ -14,8 +14,8 @@ github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtM
 github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/dataclouder/go-vcloud-director/v2 v2.17.0-alpha.3.0.20231017090632-1876e0d9beea h1:CTudwDp+nbrMnArmphxm+90aj1EtuY9ndXiAnrLhvuc=
-github.com/dataclouder/go-vcloud-director/v2 v2.17.0-alpha.3.0.20231017090632-1876e0d9beea/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw=
+github.com/dataclouder/go-vcloud-director/v2 v2.17.0-alpha.3.0.20231023153621-323907328698 h1:AzGAP35kaEG5DEaFanJZVGFBYjPIIldxY+1o+wODbww=
+github.com/dataclouder/go-vcloud-director/v2 v2.17.0-alpha.3.0.20231023153621-323907328698/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

vcd/api_test.go

@@ -151,14 +151,3 @@ func getVersionFromFile(fileName string) (string, error) {
 
 	return strings.TrimSpace(string(versionText)), nil
 }
-
-// firstNonEmpty returns the first non empty string from a list
-// If all arguments are empty, returns an empty string
-func firstNonEmpty(args ...string) string {
-	for _, s := range args {
-		if s != "" {
-			return s
-		}
-	}
-	return ""
-}

vcd/datasource_vcd_catalog_media.go

@@ -83,6 +83,11 @@ func datasourceVcdCatalogMedia() *schema.Resource {
 				Computed:    true,
 				Description: "Storage profile name",
 			},
+			"download_to_file": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Will download the contents of the media item into the given file",
+			},
 			"filter": {
 				Type:        schema.TypeList,
 				MaxItems:    1,

vcd/datasource_vcd_catalog_media_test.go

@@ -4,7 +4,9 @@ package vcd
 
 import (
 	"fmt"
+	"os"
 	"regexp"
+	"runtime"
 	"testing"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
@@ -21,6 +23,11 @@ func TestAccVcdCatalogAndMediaDatasource(t *testing.T) {
 	var TestAccVcdDataSourceMedia = "TestAccVcdCatalogMediaBasic"
 	var TestAccVcdDataSourceMediaDescription = "TestAccVcdCatalogMediaBasicDescription"
 
+	_, sourceFile, _, _ := runtime.Caller(0)
+	if !fileExists(sourceFile) {
+		t.Skip("source file for this test was not found")
+	}
+	tempFile := "source_file.txt"
 	var params = StringMap{
 		"Org":              testConfig.VCD.Org,
 		"Catalog":          testConfig.VCD.Catalog.Name,
@@ -30,8 +37,12 @@ func TestAccVcdCatalogAndMediaDatasource(t *testing.T) {
 		"UploadProgress":   testConfig.Ova.UploadProgress,
 		"Tags":             "catalog",
 		"CatalogMediaName": TestAccVcdDataSourceMedia,
+		"MediaFileName":    "TestMediaFile",
+		"MediaFilePath":    sourceFile,
+		"DownloadToFile":   tempFile,
 		"Description":      TestAccVcdDataSourceMediaDescription,
 		"MediaPath":        testConfig.Media.MediaPath,
+		"FuncName":         t.Name(),
 	}
 	testParamsNotEmpty(t, params)
 
@@ -43,6 +54,14 @@ func TestAccVcdCatalogAndMediaDatasource(t *testing.T) {
 
 	debugPrintf("#[DEBUG] CONFIGURATION: %s", configText)
 
+	defer func() {
+		if fileExists(tempFile) {
+			err := os.Remove(tempFile)
+			if err != nil {
+				fmt.Printf("error deleting file '%s': %s", tempFile, err)
+			}
+		}
+	}()
 	resource.Test(t, resource.TestCase{
 		PreCheck:          func() { preRunChecks(t) },
 		ProviderFactories: testAccProviders,
@@ -56,14 +75,37 @@ func TestAccVcdCatalogAndMediaDatasource(t *testing.T) {
 					resource.TestMatchOutput("creation_date", regexp.MustCompile(`^\d{4}-\d{2}-\d{2}.*`)),
 					resource.TestCheckOutput("status", "RESOLVED"),
 					resource.TestMatchOutput("storage_profile_name", regexp.MustCompile(`^\S+`)),
+					resource.TestCheckResourceAttr("vcd_catalog_media.media_file", "name", "TestMediaFile"),
 					testCheckMediaNonStringOutputs(),
 				),
 			},
+			{
+				Config: configText,
+				Check:  checkFileContentsAreEqual(sourceFile, tempFile),
+			},
 		},
 	})
 	postTestChecks(t)
 }
 
+func checkFileContentsAreEqual(fileName1, fileName2 string) resource.TestCheckFunc {
+
+	return func(s *terraform.State) error {
+		contents1, err := os.ReadFile(fileName1) // #nosec G304 -- file name under control used for testing
+		if err != nil {
+			return err
+		}
+		contents2, err := os.ReadFile(fileName2) // #nosec G304 -- file name under control used for testing
+		if err != nil {
+			return err
+		}
+		if string(contents1) == string(contents2) {
+			return nil
+		}
+		return fmt.Errorf("file %s and %s have different content", fileName1, fileName2)
+	}
+}
+
 func catalogMediaDestroyed(catalog, mediaName string) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		conn := testAccProvider.Meta().(*VCDClient)
@@ -84,29 +126,57 @@ func catalogMediaDestroyed(catalog, mediaName string) resource.TestCheckFunc {
 }
 
 const testAccCheckVcdCatalogMediaDS = `
+
+data "vcd_catalog" "mycat" {
+  org  = "{{.Org}}"
+  name = "{{.Catalog}}"
+}
+
 resource "vcd_catalog_media"  "{{.CatalogMediaName}}" {
-  org     = "{{.Org}}"
-  catalog = "{{.Catalog}}"
+  org        = "{{.Org}}"
+  catalog_id = data.vcd_catalog.mycat.id
 
   name                 = "{{.CatalogMediaName}}"
   description          = "{{.Description}}"
   media_path           = "{{.MediaPath}}"
   upload_piece_size    = {{.UploadPieceSize}}
-  show_upload_progress = "{{.UploadProgress}}"
+  show_upload_progress = {{.UploadProgress}}
 
   metadata = {
     catalogMedia_metadata = "catalogMedia Metadata"
     catalogMedia_metadata2 = "catalogMedia Metadata2"
   }
 }
 
+# this resource uploads the source file for the current test as a media item
+resource "vcd_catalog_media"  "media_file" {
+  org        = "{{.Org}}"
+  catalog_id = data.vcd_catalog.mycat.id
+
+  name                 = "{{.MediaFileName}}"
+  description          = "{{.Description}}"
+  media_path           = "{{.MediaFilePath}}"
+  upload_piece_size    = {{.UploadPieceSize}}
+  show_upload_progress = {{.UploadProgress}}
+  upload_any_file      = true
+}
+
 data "vcd_catalog_media" "{{.NewCatalogMedia}}" {
   org        = "{{.Org}}"
-  catalog    = "{{.Catalog}}"
+  catalog_id = data.vcd_catalog.mycat.id
   name       = vcd_catalog_media.{{.CatalogMediaName}}.name
   depends_on = [vcd_catalog_media.{{.CatalogMediaName}}]
 }
 
+# This data source downloads the contents of the media item to a local file
+data "vcd_catalog_media" "media_file_ds" {
+  org        = "{{.Org}}"
+  catalog_id = data.vcd_catalog.mycat.id
+  name       = vcd_catalog_media.media_file.name
+
+  download_to_file = "{{.DownloadToFile}}"
+}
+
 output "size" {
   value = data.vcd_catalog_media.{{.NewCatalogMedia}}.size
 }

vcd/datasource_vcd_resource_list.go

@@ -148,7 +148,7 @@ func getPvdcList(d *schema.ResourceData, meta interface{}) (list []string, err e
 func getVdcGroups(d *schema.ResourceData, meta interface{}) (list []string, err error) {
 	client := meta.(*VCDClient)
 
-	org, err := client.GetAdminOrgByName(d.Get("org").(string))
+	org, err := client.GetAdminOrg(firstNonEmpty(d.Get("org").(string), d.Get("parent").(string)))
 	if err != nil {
 		return list, err
 	}
@@ -203,7 +203,8 @@ func externalNetworkList(d *schema.ResourceData, meta interface{}) (list []strin
 
 func rightsList(d *schema.ResourceData, meta interface{}) (list []string, err error) {
 	client := meta.(*VCDClient)
-	org, err := client.GetAdminOrg(d.Get("org").(string))
+
+	org, err := client.GetAdminOrg(firstNonEmpty(d.Get("org").(string), d.Get("parent").(string)))
 	if err != nil {
 		return list, err
 	}
@@ -229,7 +230,7 @@ func rightsList(d *schema.ResourceData, meta interface{}) (list []string, err er
 func rolesList(d *schema.ResourceData, meta interface{}) (list []string, err error) {
 	client := meta.(*VCDClient)
 
-	org, err := client.GetAdminOrg(d.Get("org").(string))
+	org, err := client.GetAdminOrg(firstNonEmpty(d.Get("org").(string), d.Get("parent").(string)))
 	if err != nil {
 		return list, err
 	}
@@ -276,7 +277,7 @@ func globalRolesList(d *schema.ResourceData, meta interface{}) (list []string, e
 func libraryCertificateList(d *schema.ResourceData, meta interface{}) (list []string, err error) {
 	client := meta.(*VCDClient)
 
-	adminOrg, err := client.GetAdminOrg(d.Get("org").(string))
+	adminOrg, err := client.GetAdminOrg(firstNonEmpty(d.Get("org").(string), d.Get("parent").(string)))
 	if err != nil {
 		return list, err
 	}
@@ -332,8 +333,7 @@ func rightsBundlesList(d *schema.ResourceData, meta interface{}) (list []string,
 
 func catalogList(d *schema.ResourceData, meta interface{}, resType string) (list []string, err error) {
 	client := meta.(*VCDClient)
-
-	org, err := client.GetAdminOrg(d.Get("org").(string))
+	org, err := client.GetAdminOrg(firstNonEmpty(d.Get("org").(string), d.Get("parent").(string)))
 	if err != nil {
 		return list, err
 	}
@@ -449,7 +449,7 @@ func vappTemplateList(d *schema.ResourceData, meta interface{}) (list []string,
 func vdcList(d *schema.ResourceData, meta interface{}, resType string) (list []string, err error) {
 	client := meta.(*VCDClient)
 
-	org, err := client.GetAdminOrg(d.Get("org").(string))
+	org, err := client.GetAdminOrg(firstNonEmpty(d.Get("org").(string), d.Get("parent").(string)))
 	if err != nil {
 		return list, err
 	}
@@ -469,7 +469,7 @@ func vdcList(d *schema.ResourceData, meta interface{}, resType string) (list []s
 func orgUserList(d *schema.ResourceData, meta interface{}) (list []string, err error) {
 	client := meta.(*VCDClient)
 
-	org, err := client.GetAdminOrg(d.Get("org").(string))
+	org, err := client.GetAdminOrg(firstNonEmpty(d.Get("org").(string), d.Get("parent").(string)))
 	if err != nil {
 		return list, err
 	}

vcd/datasource_vcd_resource_list_test.go

@@ -77,12 +77,12 @@ func TestAccVcdDatasourceResourceList(t *testing.T) {
 	} else {
 		fmt.Print("`Networking.ExternalNetwork` value isn't configured, datasource test using this will be skipped\n")
 	}
-	if testConfig.VCD.ProviderVdc.Name != "" {
+	if testConfig.VCD.ProviderVdc.Name != "" && usingSysAdmin() {
 		lists = append(lists, listDef{name: "provider-vdc", resourceType: "vcd_provider_vdc", knownItem: testConfig.VCD.ProviderVdc.Name})
 	} else {
 		fmt.Print("`VCD.ProviderVdc` value isn't configured, datasource test using this will be skipped\n")
 	}
-	if testConfig.VCD.NsxtProviderVdc.Name != "" {
+	if testConfig.VCD.NsxtProviderVdc.Name != "" && usingSysAdmin() {
 		lists = append(lists, listDef{name: "nsxt-provider-vdc", resourceType: "vcd_provider_vdc", knownItem: testConfig.VCD.NsxtProviderVdc.Name})
 	} else {
 		fmt.Print("`VCD.NsxtProviderVdc` value isn't configured, datasource test using this will be skipped\n")
@@ -154,13 +154,31 @@ func TestAccVcdDatasourceResourceList(t *testing.T) {
 			"VCD.Vdc` value isn't configured, datasource test using this will be skipped\n")
 	}
 
-	if testConfig.Nsxt.VdcGroup != "" && testConfig.Nsxt.VdcGroupEdgeGateway != "" {
+	if testConfig.Nsxt.VdcGroup != "" {
+		// Retrieves the list of VDC groups, filling the "parent" field
 		lists = append(lists, listDef{
-			name:         "VdcGroupEdge",
-			resourceType: "vcd_nsxt_edgegateway",
-			parent:       testConfig.Nsxt.VdcGroup,
-			knownItem:    testConfig.Nsxt.VdcGroupEdgeGateway,
+			name:         "VdcGroupAsParent",
+			resourceType: "vcd_vdc_group",
+			parent:       testConfig.VCD.Org,
+			knownItem:    testConfig.Nsxt.VdcGroup,
+		})
+
+		// Retrieves the list of VDC groups, filling only the "org" field through the provider
+		lists = append(lists, listDef{
+			name:         "VdcGroupAsOrg",
+			resourceType: "vcd_vdc_group",
+			knownItem:    testConfig.Nsxt.VdcGroup,
 		})
+
+		// Retrieves the list of NSX-T edge gateway belonging to a VDC group
+		if testConfig.Nsxt.VdcGroupEdgeGateway != "" {
+			lists = append(lists, listDef{
+				name:         "VdcGroupEdge",
+				resourceType: "vcd_nsxt_edgegateway",
+				parent:       testConfig.Nsxt.VdcGroup,
+				knownItem:    testConfig.Nsxt.VdcGroupEdgeGateway,
+			})
+		}
 	}
 
 	if testConfig.Nsxt.Vdc != "" {

vcd/resource_vcd_catalog_media.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
+	"path"
 	"strings"
 	"time"
 
@@ -66,6 +68,12 @@ func resourceVcdCatalogMedia() *schema.Resource {
 				ForceNew:    true,
 				Description: "absolute or relative path to Media file",
 			},
+			"upload_any_file": {
+				Type:        schema.TypeBool,
+				Optional:    true,
+				Default:     false,
+				Description: "If true, will allow uploading any file type, not only .ISO",
+			},
 			"upload_piece_size": {
 				Type:        schema.TypeInt,
 				Optional:    true,
@@ -157,7 +165,14 @@ func resourceVcdMediaCreate(ctx context.Context, d *schema.ResourceData, meta in
 
 	uploadPieceSize := d.Get("upload_piece_size").(int)
 	mediaName := d.Get("name").(string)
-	task, err := catalog.UploadMediaImage(mediaName, d.Get("description").(string), mediaPath, int64(uploadPieceSize)*1024*1024) // Convert from megabytes to bytes)
+	var task govcd.UploadTask
+	uploadAnyFile := d.Get("upload_any_file").(bool)
+
+	if uploadAnyFile {
+		task, err = catalog.UploadMediaFile(mediaName, d.Get("description").(string), mediaPath, int64(uploadPieceSize)*1024*1024, false) // Convert from megabytes to bytes)
+	} else {
+		task, err = catalog.UploadMediaImage(mediaName, d.Get("description").(string), mediaPath, int64(uploadPieceSize)*1024*1024) // Convert from megabytes to bytes)
+	}
 	if err != nil {
 		log.Printf("Error uploading new catalog media: %s", err)
 		return diag.Errorf("error uploading new catalog media: %s", err)
@@ -290,6 +305,20 @@ func genericVcdMediaRead(d *schema.ResourceData, meta interface{}, origin string
 	dSet(d, "status", mediaRecord.MediaRecord.Status)
 	dSet(d, "storage_profile_name", mediaRecord.MediaRecord.StorageProfileName)
 
+	if origin == "datasource" {
+		downloadToFile := d.Get("download_to_file").(string)
+		if downloadToFile != "" {
+			contents, err := media.Download()
+			if err != nil {
+				return diag.Errorf("error downloading media contents")
+			}
+			downloadToFile = path.Clean(downloadToFile)
+			err = os.WriteFile(downloadToFile, contents, 0600)
+			if err != nil {
+				return diag.Errorf("error writing media contents to file '%s'", downloadToFile)
+			}
+		}
+	}
 	diagErr := updateMetadataInState(d, vcdClient, "vcd_catalog_media", media)
 	if diagErr != nil {
 		log.Printf("[DEBUG] Unable to update media item metadata: %s", err)