MTV-1388 | Add dynamic way to specify the virt-customize

Issue:
Right now all the run/firstboot scripts are located inside the
conversion pod. To get new scripts to the users we need some alternative
way to allow users to specify the scripts themselves. This will speed
up the migration process as the users won't need to wait for the build
and release of the patch. Additionally, the users themselves can tweak
the scripts depending on their needs.

Design:
We have two options use the config maps, mount it to the
container and read the mounted directory depending on their name we
do virt-customize with them. An alternative solution would be to create CRD
with all configurations such as `action`, `path` etc. I have chosen first
as this is needed as soon as possible.

User flow:
1. The user needs to create a config map inside the namespace where they want to
   migrate the VM. The config map needs to be named `mtv-virt-customize`.
2. Inside the configmap user can specify data with key/value definition.
   The value is script itself which they want to run.
   The key needs to follow regex `^([0-9]+_win_firstboot(([\w\-]*).ps1))$` or
   `^([0-9]+_linux_(run|firstboot)(([\w\-]*).sh))$` depending on where the customer
   wants the script to run.
   For example `00_win_firstboot_test.ps1` will specify that the data
   should be interpreted as PowerShell script which should start at boot.
   Alternatively, the Linux option has not only the `firstboot` but also
   the `run` option. This will be applied on the VM after virt-v2v conversion,
   but before the VM is started.
   Note: The number in the begining of the key sets the order.

Conversion pod flow:
1. The `forklift-controller` checks if there is a config map inside the
   namespace. If the config map is present it is automatically mounted to
   the conversion pod at `/mnt/dynamic_scripts`.
2. The conversion pod checks if the `/mnt/dynamic_scripts` directory is
   present and if it is, it checks the files and their regexes.
   Depending on the VM operating system it will use different. From the
   filename the conversion pod determines the action required on the
   `virt-customize` step.

Additional changes:
- I have moved the static variables into a single file as we are using
  more and more and it's hard to keep track.
- New batch file which will run all PowerShell scripts inside the
  windows first boot scripts dir.

Signed-off-by: Martin Necas <[email protected]>

operator/config/manager/manager.yaml

@@ -76,6 +76,8 @@ spec:
           value: ${OVIRT_OS_MAP}
         - name: VSPHERE_OS_MAP
           value: ${VSPHERE_OS_MAP}
+        - name: VIRT_CUSTOMIZE_MAP
+          value: ${VIRT_CUSTOMIZE_MAP}
         livenessProbe:
           httpGet:
             path: /healthz

operator/roles/forkliftcontroller/defaults/main.yml

@@ -22,6 +22,7 @@ controller_configmap_name: "{{ controller_service_name }}-config"
 controller_service_name: "{{ app_name }}-controller"
 ovirt_osmap_configmap_name: "forklift-ovirt-osmap"
 vsphere_osmap_configmap_name: "forklift-vsphere-osmap"
+virt_customize_configmap_name: "{{ app_name }}-virt-customize"
 controller_deployment_name: "{{ controller_service_name }}"
 controller_container_name: "{{ app_name }}-controller"
 controller_container_limits_cpu: "500m"

operator/roles/forkliftcontroller/templates/controller/deployment-controller.yml.j2

@@ -111,6 +111,10 @@ spec:
         - name: VSPHERE_OS_MAP
           value: {{ vsphere_osmap_configmap_name }}
 {% endif %}
+{% if virt_customize_configmap_name is defined %}
+        - name: VIRT_CUSTOMIZE_MAP
+          value: {{ virt_customize_configmap_name }}
+{% endif %}
 {% if controller_profile_kind is defined and controller_profile_path is defined and controller_profile_duration is defined %}
         - name: PROFILE_KIND
           value: "{{ controller_profile_kind }}"

pkg/controller/plan/kubevirt.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	k8svalidation "k8s.io/apimachinery/pkg/util/validation"
@@ -68,8 +69,10 @@ const (
 	// DV deletion on completion
 	AnnDeleteAfterCompletion = "cdi.kubevirt.io/storage.deleteAfterCompletion"
 	// Max Length for vm name
-	NameMaxLength  = 63
-	VddkVolumeName = "vddk-vol-mount"
+	NameMaxLength            = 63
+	VddkVolumeName           = "vddk-vol-mount"
+	DynamicScriptsVolumeName = "scripts-volume-mount"
+	DynamicScriptsMountPath  = "/mnt/dynamic_scripts"
 )
 
 // Labels
@@ -1949,6 +1952,28 @@ func (r *KubeVirt) podVolumeMounts(vmVolumes []cnv.Volume, configMap *core.Confi
 		}
 	}
 
+	_, exists, err := r.findConfigMapInNamespace(Settings.VirtCustomizeConfigMap, r.Plan.Spec.TargetNamespace)
+	if err != nil {
+		err = liberr.Wrap(err)
+		return
+	}
+	if exists {
+		volumes = append(volumes, core.Volume{
+			Name: DynamicScriptsVolumeName,
+			VolumeSource: core.VolumeSource{
+				ConfigMap: &core.ConfigMapVolumeSource{
+					LocalObjectReference: core.LocalObjectReference{
+						Name: Settings.VirtCustomizeConfigMap,
+					},
+				},
+			},
+		})
+		mounts = append(mounts, core.VolumeMount{
+			Name:      DynamicScriptsVolumeName,
+			MountPath: DynamicScriptsMountPath,
+		})
+	}
+
 	// Temporary space for VDDK library
 	volumes = append(volumes, core.Volume{
 		Name: VddkVolumeName,
@@ -2058,6 +2083,33 @@ func (r *KubeVirt) libvirtDomain(vmCr *VirtualMachine, pvcs []*core.PersistentVo
 	return
 }
 
+func (r *KubeVirt) findConfigMapInNamespace(name string, namespace string) (configMap *core.ConfigMap, exists bool, err error) {
+	if err != nil {
+		return
+	}
+	list := &core.ConfigMapList{}
+	nameField := "metadata.name"
+	namespaceField := "metadata.namespace"
+	err = r.Destination.Client.List(
+		context.TODO(),
+		list,
+		&client.ListOptions{
+			FieldSelector: fields.SelectorFromSet(map[string]string{
+				nameField:      name,
+				namespaceField: namespace,
+			}),
+		},
+	)
+	if err != nil {
+		err = liberr.Wrap(err)
+		return
+	}
+	if len(list.Items) > 0 {
+		return &list.Items[0], true, nil
+	}
+	return nil, false, nil
+}
+
 // Ensure the config map exists on the destination.
 func (r *KubeVirt) ensureConfigMap(vmRef ref.Ref) (configMap *core.ConfigMap, err error) {
 	_, err = r.Source.Inventory.VM(&vmRef)

pkg/settings/migration.go

@@ -26,6 +26,7 @@ const (
 	CleanupRetries            = "CLEANUP_RETRIES"
 	OvirtOsConfigMap          = "OVIRT_OS_MAP"
 	VsphereOsConfigMap        = "VSPHERE_OS_MAP"
+	VirtCustomizeConfigMap    = "VIRT_CUSTOMIZE_MAP"
 	VddkJobActiveDeadline     = "VDDK_JOB_ACTIVE_DEADLINE"
 	VirtV2vExtraArgs          = "VIRT_V2V_EXTRA_ARGS"
 	VirtV2vExtraConfConfigMap = "VIRT_V2V_EXTRA_CONF_CONFIG_MAP"
@@ -61,6 +62,8 @@ type Migration struct {
 	OvirtOsConfigMap string
 	// vSphere OS config map name
 	VsphereOsConfigMap string
+	// vSphere OS config map name
+	VirtCustomizeConfigMap string
 	// Active deadline for VDDK validation job
 	VddkJobActiveDeadline int
 	// Additional arguments for virt-v2v
@@ -89,6 +92,11 @@ func (r *Migration) Load() (err error) {
 	if r.SnapshotStatusCheckRate, err = getPositiveEnvLimit(SnapshotStatusCheckRate, 10); err != nil {
 		return liberr.Wrap(err)
 	}
+	if virtCustomizeConfigMap, ok := os.LookupEnv(VirtCustomizeConfigMap); ok {
+		r.VirtCustomizeConfigMap = virtCustomizeConfigMap
+	} else if Settings.Role.Has(MainRole) {
+		return liberr.Wrap(fmt.Errorf("failed to find environment variable %s", VirtCustomizeConfigMap))
+	}
 	if r.CleanupRetries, err = getPositiveEnvLimit(CleanupRetries, 10); err != nil {
 		return liberr.Wrap(err)
 	}

virt-v2v/pkg/customize/BUILD.bazel

@@ -6,13 +6,14 @@ go_library(
         "image.go",
         "rhel.go",
         "windows.go",
+        "utils.go",
     ],
     embedsrcs = [
         "scripts/rhel/firstboot/README.md",
         "scripts/rhel/run/README.md",
         "scripts/rhel/run/network_config_util.sh",
         "scripts/windows/9999-restore_config.ps1",
-        "scripts/windows/9999-restore_config_init.bat",
+        "scripts/windows/9999-run-mtv-ps-scripts.bat",
         "scripts/windows/firstboot.bat",
         "scripts/rhel/run/ifcfg-double-quotes-test.d/expected-udev.rule",
         "scripts/rhel/run/ifcfg-double-quotes-test.d/root/etc/sysconfig/network-scripts/ifcfg-eth0",

virt-v2v/pkg/customize/image.go

@@ -22,6 +22,7 @@ type DomainExecFunc func(args ...string) error
 
 func Run(disks []string, operatingSystem string) error {
 	var err error
+	fmt.Printf("Customizing disks '%s'\n", disks)
 	// Customization for vSphere source.
 	t := utils.EmbedTool{Filesystem: &scriptFS}
 	// windows

virt-v2v/pkg/customize/rhel.go

@@ -2,16 +2,16 @@ package customize
 
 import (
 	"fmt"
+	"github.com/konveyor/forklift-controller/virt-v2v/pkg/global"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/konveyor/forklift-controller/virt-v2v/pkg/utils"
 )
 
 func CustomizeLinux(execFunc DomainExecFunc, disks []string, dir string, t FileSystemTool) error {
-	fmt.Printf("Customizing disks '%v'\n", disks)
-
 	var extraArgs []string
 
 	// Step 1: Create files from the filesystem
@@ -24,11 +24,20 @@ func CustomizeLinux(execFunc DomainExecFunc, disks []string, dir string, t FileS
 		return err
 	}
 
-	// Step 3: Add scripts
-	if err := addRunScripts(&extraArgs, dir); err != nil {
+	// Step 3: Add dynamic scripts from the configmap
+
+	if _, err := os.Stat(global.DYNAMIC_SCRIPTS_MOUNT_PATH); !os.IsNotExist(err) {
+		fmt.Println("Adding linux dynamic scripts")
+		if err = addRhelDynamicScripts(&extraArgs, global.DYNAMIC_SCRIPTS_MOUNT_PATH); err != nil {
+			return err
+		}
+	}
+
+	// Step 3: Add scripts from embeded FS
+	if err := addRhelRunScripts(&extraArgs, dir); err != nil {
 		return err
 	}
-	if err := addFirstbootScripts(&extraArgs, dir); err != nil {
+	if err := addRhelFirstbootScripts(&extraArgs, dir); err != nil {
 		return err
 	}
 
@@ -65,11 +74,11 @@ func handleStaticIPConfiguration(extraArgs *[]string, dir string) error {
 	return nil
 }
 
-// addFirstbootScripts appends firstboot script arguments to extraArgs
-func addFirstbootScripts(extraArgs *[]string, dir string) error {
+// addRhelFirstbootScripts appends firstboot script arguments to extraArgs
+func addRhelFirstbootScripts(extraArgs *[]string, dir string) error {
 	firstbootScriptsPath := filepath.Join(dir, "scripts", "rhel", "firstboot")
 
-	firstBootScripts, err := getScripts(firstbootScriptsPath)
+	firstBootScripts, err := getScriptsWithSuffix(firstbootScriptsPath, global.SHELL_SUFFIX)
 	if err != nil {
 		return err
 	}
@@ -83,11 +92,11 @@ func addFirstbootScripts(extraArgs *[]string, dir string) error {
 	return nil
 }
 
-// addRunScripts appends run script arguments to extraArgs
-func addRunScripts(extraArgs *[]string, dir string) error {
+// addRhelRunScripts appends run script arguments to extraArgs
+func addRhelRunScripts(extraArgs *[]string, dir string) error {
 	runScriptsPath := filepath.Join(dir, "scripts", "rhel", "run")
 
-	runScripts, err := getScripts(runScriptsPath)
+	runScripts, err := getScriptsWithSuffix(runScriptsPath, global.SHELL_SUFFIX)
 	if err != nil {
 		return err
 	}
@@ -101,29 +110,6 @@ func addRunScripts(extraArgs *[]string, dir string) error {
 	return nil
 }
 
-// getScripts retrieves all .sh scripts from the specified directory
-func getScripts(directory string) ([]string, error) {
-	files, err := os.ReadDir(directory)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read firstboot scripts directory: %w", err)
-	}
-
-	var scripts []string
-	for _, file := range files {
-		if !file.IsDir() && strings.HasSuffix(file.Name(), ".sh") && !strings.HasPrefix(file.Name(), "test-") {
-			scriptPath := filepath.Join(directory, file.Name())
-			scripts = append(scripts, scriptPath)
-		}
-	}
-
-	return scripts, nil
-}
-
-// addDisksToCustomize appends disk arguments to extraArgs
-func addDisksToCustomize(extraArgs *[]string, disks []string) {
-	*extraArgs = append(*extraArgs, utils.GetScriptArgs("add", disks...)...)
-}
-
 // addLuksKeysToCustomize appends key arguments to extraArgs
 func addLuksKeysToCustomize(extraArgs *[]string) error {
 	luksArgs, err := utils.AddLUKSKeys()
@@ -134,3 +120,19 @@ func addLuksKeysToCustomize(extraArgs *[]string) error {
 
 	return nil
 }
+
+func addRhelDynamicScripts(extraArgs *[]string, dir string) error {
+	dynamicScripts, err := getScriptsWithRegex(dir, global.LINUX_DYNAMIC_REGEX)
+	if err != nil {
+		return err
+	}
+	for _, script := range dynamicScripts {
+		fmt.Printf("Adding linux dynamic scripts '%s'\n", script)
+		r := regexp.MustCompile(global.LINUX_DYNAMIC_REGEX)
+		groups := r.FindStringSubmatch(filepath.Base(script))
+		// Option from the second regex group `(run|firstboot)`
+		action := groups[2]
+		*extraArgs = append(*extraArgs, utils.GetScriptArgs(action, script)...)
+	}
+	return nil
+}

virt-v2v/pkg/customize/rhel_test.go

@@ -2,6 +2,7 @@ package customize
 
 import (
 	"fmt"
+	"github.com/konveyor/forklift-controller/virt-v2v/pkg/global"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -125,9 +126,9 @@ func TestAddFirstbootScripts(t *testing.T) {
 	}
 
 	extraArgs := []string{}
-	err = addFirstbootScripts(&extraArgs, tempDir)
+	err = addRhelFirstbootScripts(&extraArgs, tempDir)
 	if err != nil {
-		t.Fatalf("addFirstbootScripts returned an error: %v", err)
+		t.Fatalf("addRhelFirstbootScripts returned an error: %v", err)
 	}
 
 	if len(extraArgs) == 0 || !contains(extraArgs, "--firstboot") {
@@ -148,9 +149,9 @@ func TestAddRunScripts(t *testing.T) {
 	}
 
 	extraArgs := []string{}
-	err = addRunScripts(&extraArgs, tempDir)
+	err = addRhelRunScripts(&extraArgs, tempDir)
 	if err != nil {
-		t.Fatalf("addRunScripts returned an error: %v", err)
+		t.Fatalf("addRhelRunScripts returned an error: %v", err)
 	}
 
 	if len(extraArgs) == 0 || !contains(extraArgs, "--run") {
@@ -170,17 +171,17 @@ func TestGetScripts(t *testing.T) {
 		t.Fatalf("Error WriteFile: %v", err)
 	}
 
-	scripts, err := getScripts(tempDir)
+	scripts, err := getScriptsWithSuffix(tempDir, global.SHELL_SUFFIX)
 	if err != nil {
-		t.Fatalf("getScripts returned an error: %v", err)
+		t.Fatalf("getScriptsWithSuffix returned an error: %v", err)
 	}
 
 	expectedScripts := []string{
 		filepath.Join(tempDir, "test1.sh"),
 		filepath.Join(tempDir, "test2.sh"),
 	}
 	if !reflect.DeepEqual(scripts, expectedScripts) {
-		t.Fatalf("getScripts returned incorrect scripts: got %v, want %v", scripts, expectedScripts)
+		t.Fatalf("getScriptsWithSuffix returned incorrect scripts: got %v, want %v", scripts, expectedScripts)
 	}
 }
 
@@ -213,9 +214,9 @@ func TestAddFirstbootScripts_NoScripts(t *testing.T) {
 	}
 
 	extraArgs := []string{}
-	err = addFirstbootScripts(&extraArgs, tempDir)
+	err = addRhelFirstbootScripts(&extraArgs, tempDir)
 	if err != nil {
-		t.Fatalf("addFirstbootScripts returned an error: %v", err)
+		t.Fatalf("addRhelFirstbootScripts returned an error: %v", err)
 	}
 
 	// Ensure no "--firstboot" argument is added when no scripts are found
@@ -233,9 +234,9 @@ func TestAddRunScripts_NoScripts(t *testing.T) {
 	}
 
 	extraArgs := []string{}
-	err = addRunScripts(&extraArgs, tempDir)
+	err = addRhelRunScripts(&extraArgs, tempDir)
 	if err != nil {
-		t.Fatalf("addRunScripts returned an error: %v", err)
+		t.Fatalf("addRhelRunScripts returned an error: %v", err)
 	}
 
 	// Ensure no "--run" argument is added when no scripts are found
@@ -283,9 +284,9 @@ func TestAddFirstbootScripts_ReadDirFails(t *testing.T) {
 	tempDir := "/invalid-dir"
 
 	extraArgs := []string{}
-	err := addFirstbootScripts(&extraArgs, tempDir)
+	err := addRhelFirstbootScripts(&extraArgs, tempDir)
 	if err == nil {
-		t.Fatalf("Expected error in addFirstbootScripts due to read failure, got nil")
+		t.Fatalf("Expected error in addRhelFirstbootScripts due to read failure, got nil")
 	}
 }
 
@@ -294,8 +295,8 @@ func TestAddRunScripts_ReadDirFails(t *testing.T) {
 	tempDir := "/invalid-dir"
 
 	extraArgs := []string{}
-	err := addRunScripts(&extraArgs, tempDir)
+	err := addRhelRunScripts(&extraArgs, tempDir)
 	if err == nil {
-		t.Fatalf("Expected error in addRunScripts due to read failure, got nil")
+		t.Fatalf("Expected error in addRhelRunScripts due to read failure, got nil")
 	}
 }