add an env var to toggle gzip compression for cloud-init data (#531)

cmd/main.go

@@ -271,12 +271,18 @@ func setupControllers(mgr manager.Manager, flags flagVars, linodeClientConfig, d
 		os.Exit(1)
 	}
 
+	useGzip, err := strconv.ParseBool(os.Getenv("GZIP_COMPRESSION_ENABLED"))
+	if err != nil {
+		setupLog.Error(err, "proceeding without gzip compression for cloud-init data")
+	}
+
 	// LinodeMachine Controller
 	if err := (&controller.LinodeMachineReconciler{
-		Client:             mgr.GetClient(),
-		Recorder:           mgr.GetEventRecorderFor("LinodeMachineReconciler"),
-		WatchFilterValue:   flags.machineWatchFilter,
-		LinodeClientConfig: linodeClientConfig,
+		Client:                 mgr.GetClient(),
+		Recorder:               mgr.GetEventRecorderFor("LinodeMachineReconciler"),
+		WatchFilterValue:       flags.machineWatchFilter,
+		LinodeClientConfig:     linodeClientConfig,
+		GzipCompressionEnabled: useGzip,
 	}).SetupWithManager(mgr, crcontroller.Options{MaxConcurrentReconciles: flags.linodeMachineConcurrency}); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "LinodeMachine")
 		os.Exit(1)

controller/linodemachine_controller.go

@@ -95,6 +95,8 @@ type LinodeMachineReconciler struct {
 	LinodeClientConfig scope.ClientConfig
 	WatchFilterValue   string
 	ReconcileTimeout   time.Duration
+	// Feature flags
+	GzipCompressionEnabled bool
 }
 
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=linodemachines,verbs=get;list;watch;create;update;patch;delete
@@ -400,7 +402,7 @@ func (r *LinodeMachineReconciler) reconcilePreflightMetadataSupportConfigure(ctx
 
 func (r *LinodeMachineReconciler) reconcilePreflightCreate(ctx context.Context, logger logr.Logger, machineScope *scope.MachineScope) (ctrl.Result, error) {
 	// get the bootstrap data for the Linode instance and set it for create config
-	createOpts, err := newCreateConfig(ctx, machineScope, logger)
+	createOpts, err := newCreateConfig(ctx, machineScope, r.GzipCompressionEnabled, logger)
 	if err != nil {
 		logger.Error(err, "Failed to create Linode machine InstanceCreateOptions")
 		return retryIfTransient(err, logger)

controller/linodemachine_controller_helpers.go

@@ -17,6 +17,8 @@ limitations under the License.
 package controller
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
 	b64 "encoding/base64"
 	"errors"
@@ -97,7 +99,7 @@ func fillCreateConfig(createConfig *linodego.InstanceCreateOptions, machineScope
 	}
 }
 
-func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, logger logr.Logger) (*linodego.InstanceCreateOptions, error) {
+func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, gzipCompressionEnabled bool, logger logr.Logger) (*linodego.InstanceCreateOptions, error) {
 	var err error
 
 	createConfig := linodeMachineSpecToInstanceCreateConfig(machineScope.LinodeMachine.Spec)
@@ -110,7 +112,7 @@ func newCreateConfig(ctx context.Context, machineScope *scope.MachineScope, logg
 	}
 
 	createConfig.Booted = util.Pointer(false)
-	if err := setUserData(ctx, machineScope, createConfig, logger); err != nil {
+	if err := setUserData(ctx, machineScope, createConfig, gzipCompressionEnabled, logger); err != nil {
 		return nil, err
 	}
 
@@ -478,16 +480,40 @@ func linodeMachineSpecToInstanceCreateConfig(machineSpec infrav1alpha2.LinodeMac
 	}
 }
 
-func setUserData(ctx context.Context, machineScope *scope.MachineScope, createConfig *linodego.InstanceCreateOptions, logger logr.Logger) error {
+func compressUserData(bootstrapData []byte) ([]byte, error) {
+	var userDataBuff bytes.Buffer
+	var err error
+	gz := gzip.NewWriter(&userDataBuff)
+	defer func(gz *gzip.Writer) {
+		err = gz.Close()
+	}(gz)
+	if _, err := gz.Write(bootstrapData); err != nil {
+		return nil, err
+	}
+	err = gz.Close()
+	return userDataBuff.Bytes(), err
+}
+
+func setUserData(ctx context.Context, machineScope *scope.MachineScope, createConfig *linodego.InstanceCreateOptions, gzipCompressionEnabled bool, logger logr.Logger) error {
 	bootstrapData, err := machineScope.GetBootstrapData(ctx)
 	if err != nil {
 		logger.Error(err, "Failed to get bootstrap data")
 
 		return err
 	}
 
+	userData := bootstrapData
+	//nolint:nestif // this is a temp flag until cloud-init is updated in cloud-init compatible images
 	if machineScope.LinodeMachine.Status.CloudinitMetadataSupport {
-		bootstrapSize := len(bootstrapData)
+		if gzipCompressionEnabled {
+			userData, err = compressUserData(bootstrapData)
+			if err != nil {
+				logger.Error(err, "failed to compress bootstrap data")
+
+				return err
+			}
+		}
+		bootstrapSize := len(userData)
 		if bootstrapSize > maxBootstrapDataBytesCloudInit {
 			err = errors.New("bootstrap data too large")
 			logger.Error(err, "decoded bootstrap data exceeds size limit",
@@ -498,7 +524,7 @@ func setUserData(ctx context.Context, machineScope *scope.MachineScope, createCo
 			return err
 		}
 		createConfig.Metadata = &linodego.InstanceMetadataOptions{
-			UserData: b64.StdEncoding.EncodeToString(bootstrapData),
+			UserData: b64.StdEncoding.EncodeToString(userData),
 		}
 	} else {
 		logger.Info("using StackScripts for bootstrapping")

controller/linodemachine_controller_helpers_test.go

@@ -1,14 +1,18 @@
 package controller
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
+	"crypto/rand"
 	b64 "encoding/base64"
 	"fmt"
 	"testing"
 
 	"github.com/go-logr/logr"
 	"github.com/linode/linodego"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -64,6 +68,20 @@ func TestLinodeMachineSpecToCreateInstanceConfig(t *testing.T) {
 func TestSetUserData(t *testing.T) {
 	t.Parallel()
 
+	userData := []byte("test-data")
+	if gzipCompressionFlag {
+		var userDataBuff bytes.Buffer
+		gz := gzip.NewWriter(&userDataBuff)
+		_, err = gz.Write([]byte("test-data"))
+		err = gz.Close()
+		require.NoError(t, err, "Failed to compress bootstrap data")
+		userData = userDataBuff.Bytes()
+	}
+
+	largeData := make([]byte, maxBootstrapDataBytesCloudInit*10)
+	_, err = rand.Read(largeData)
+	require.NoError(t, err, "Failed to create bootstrap data")
+
 	tests := []struct {
 		name          string
 		machineScope  *scope.MachineScope
@@ -92,7 +110,7 @@ func TestSetUserData(t *testing.T) {
 			}},
 			createConfig: &linodego.InstanceCreateOptions{},
 			wantConfig: &linodego.InstanceCreateOptions{Metadata: &linodego.InstanceMetadataOptions{
-				UserData: b64.StdEncoding.EncodeToString([]byte("test-data")),
+				UserData: b64.StdEncoding.EncodeToString(userData),
 			}},
 			expects: func(mockClient *mock.MockLinodeClient, kMock *mock.MockK8sClient) {
 				kMock.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, key types.NamespacedName, obj *corev1.Secret, opts ...client.GetOption) error {
@@ -169,7 +187,7 @@ func TestSetUserData(t *testing.T) {
 				kMock.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, key types.NamespacedName, obj *corev1.Secret, opts ...client.GetOption) error {
 					cred := corev1.Secret{
 						Data: map[string][]byte{
-							"value": make([]byte, maxBootstrapDataBytesCloudInit+1),
+							"value": largeData,
 						},
 					}
 					*obj = cred
@@ -289,7 +307,7 @@ func TestSetUserData(t *testing.T) {
 			testcase.expects(mockClient, mockK8sClient)
 			logger := logr.Logger{}
 
-			err := setUserData(context.Background(), testcase.machineScope, testcase.createConfig, logger)
+			err := setUserData(context.Background(), testcase.machineScope, testcase.createConfig, gzipCompressionFlag, logger)
 			if testcase.expectedError != nil {
 				assert.ErrorContains(t, err, testcase.expectedError.Error())
 			} else {

controller/linodemachine_controller_test.go

@@ -49,7 +49,10 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-const defaultNamespace = "default"
+const (
+	defaultNamespace    = "default"
+	gzipCompressionFlag = false
+)
 
 var _ = Describe("create", Label("machine", "create"), func() {
 	var machine clusterv1.Machine
@@ -1944,7 +1947,7 @@ var _ = Describe("machine in PlacementGroup", Label("machine", "placementGroup")
 		Expect(err).NotTo(HaveOccurred())
 		mScope.PatchHelper = patchHelper
 
-		createOpts, err := newCreateConfig(ctx, &mScope, logger)
+		createOpts, err := newCreateConfig(ctx, &mScope, gzipCompressionFlag, logger)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(createOpts).NotTo(BeNil())
 		Expect(createOpts.PlacementGroup.ID).To(Equal(1))
@@ -2121,7 +2124,7 @@ var _ = Describe("machine in VPC", Label("machine", "VPC"), Ordered, func() {
 		Expect(err).NotTo(HaveOccurred())
 		mScope.PatchHelper = patchHelper
 
-		createOpts, err := newCreateConfig(ctx, &mScope, logger)
+		createOpts, err := newCreateConfig(ctx, &mScope, gzipCompressionFlag, logger)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(createOpts).NotTo(BeNil())
 		Expect(createOpts.Interfaces).To(Equal([]linodego.InstanceConfigInterfaceCreateOptions{
@@ -2197,7 +2200,7 @@ var _ = Describe("machine in VPC", Label("machine", "VPC"), Ordered, func() {
 		Expect(err).NotTo(HaveOccurred())
 		mScope.PatchHelper = patchHelper
 
-		createOpts, err := newCreateConfig(ctx, &mScope, logger)
+		createOpts, err := newCreateConfig(ctx, &mScope, gzipCompressionFlag, logger)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(createOpts).NotTo(BeNil())
 		Expect(createOpts.Interfaces).To(Equal([]linodego.InstanceConfigInterfaceCreateOptions{