Add support for loading UTF16 encoded files

If they have a correct BOM (Byte-Order-Mark https://en.wikipedia.org/wiki/Byte_order_mark) at the beginning of the file, support non utf8 files. Falls back on UTF8 encoding, if no understandable BOM is present. Signed-off-by: Soule BA <[email protected]>
fluxcd · Sep 11, 2023 · 253e316 · 253e316
1 parent eedb1a0
commit 253e316
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 5 deletions.
diff --git a/kustomize/go.mod b/kustomize/go.mod
@@ -15,12 +15,13 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/onsi/gomega v1.27.10
 	github.com/otiai10/copy v1.12.0
+	golang.org/x/text v0.13.0
 	k8s.io/api v0.27.4
 	k8s.io/apiextensions-apiserver v0.27.4
 	k8s.io/apimachinery v0.27.4
 	k8s.io/client-go v0.27.4
 	sigs.k8s.io/controller-runtime v0.15.1
-	sigs.k8s.io/kustomize/api v0.13.4
+	sigs.k8s.io/kustomize/api v0.14.0
 	sigs.k8s.io/kustomize/kyaml v0.14.2
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -72,7 +73,6 @@ require (
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sys v0.10.0 // indirect
 	golang.org/x/term v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect

diff --git a/kustomize/go.sum b/kustomize/go.sum
@@ -243,8 +243,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

diff --git a/kustomize/kustomize_generator.go b/kustomize/kustomize_generator.go
@@ -27,6 +27,8 @@ import (
 	"sync"
 
 	securefs "github.com/fluxcd/pkg/kustomize/filesys"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -510,7 +512,16 @@ func scanManifests(fs filesys.FileSystem, base string) ([]string, error) {
 			return
 		}
 
-		fContents, err := fs.ReadFile(path)
+		f, err := os.Open(path)
+		if err != nil {
+			walkErr = err
+			return
+		}
+		defer f.Close()
+
+		utf16bom := unicode.BOMOverride(unicode.UTF8.NewDecoder())
+		reader := transform.NewReader(f, utf16bom)
+		fContents, err := io.ReadAll(reader)
 		if err != nil {
 			walkErr = err
 			return

diff --git a/kustomize/kustomize_generator_whitebox_test.go b/kustomize/kustomize_generator_whitebox_test.go
@@ -49,6 +49,14 @@ func TestScanManifests(t *testing.T) {
 			base:    "./testdata/nokustomization/panic",
 			wantErr: true,
 		},
+		{
+			name: "utf-16LE with BOM files - should be valid",
+			base: "./testdata/nokustomization/utf16le",
+			wantPaths: []string{
+				"testdata/nokustomization/utf16le/configmap.yaml",
+				"testdata/nokustomization/utf16le/secret.yaml",
+			},
+		},
 	}
 
 	for _, tt := range tests {

diff --git a/kustomize/testdata/nokustomization/utf16le/configmap.yaml b/kustomize/testdata/nokustomization/utf16le/configmap.yaml
diff --git a/kustomize/testdata/nokustomization/utf16le/secret.yaml b/kustomize/testdata/nokustomization/utf16le/secret.yaml