From 89f6cdc262b1b98143b9d2744e2db7e3b4db7f16 Mon Sep 17 00:00:00 2001 From: Shruthi Kumar Date: Tue, 10 Oct 2023 15:42:46 -0700 Subject: [PATCH] Update `rad bicep` to pull binaries from GHCR (#6426) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description We're switching to upload bicep binaries to GHCR. This updates the `rad bicep download` command to pull from GHCR instead of the existing Azure blob storage. ## Type of change - This pull request adds or changes features of Radius and has an approved issue #6353. Fixes: #6353 ## Auto-generated summary ### 🤖 Generated by Copilot at eb0ec9d ### Summary 🔄🐳🗑️ Refactored the `bicep` and `tools` packages to use `oras` to pull the bicep binary from a container registry instead of downloading it from a web server. This improves the reliability and security of the bicep installation process. Updated the tests to reflect the new download URI format and binary name. > _To pull bicep from a registry_ > _We changed the code in the bicep package_ > _We used oras to fetch_ > _And removed net/http_ > _And updated the tools and the test logic_ ### Walkthrough * Remove `net/http` package and use `oras` package to download bicep binary from container registry ([link](https://github.com/radius-project/radius/pull/6426/files?diff=unified&w=0#diff-23111ea5ee16104ce6f0692d59244d7fdca32266bac1d9bbbe93a9941dcfbd0aL21), [link](https://github.com/radius-project/radius/pull/6426/files?diff=unified&w=0#diff-0bb0df6a87062e6fad23c5f50750cb65e5e6e03112e526341f38a0e42ee06d6bL20-R34), [link](https://github.com/radius-project/radius/pull/6426/files?diff=unified&w=0#diff-23111ea5ee16104ce6f0692d59244d7fdca32266bac1d9bbbe93a9941dcfbd0aL106-R104), [link](https://github.com/radius-project/radius/pull/6426/files?diff=unified&w=0#diff-0bb0df6a87062e6fad23c5f50750cb65e5e6e03112e526341f38a0e42ee06d6bL132-R210)) * Modify `DownloadBicep` function in `bicep.go` to use new download URI format string and remove binary name parameter ([link](https://github.com/radius-project/radius/pull/6426/files?diff=unified&w=0#diff-23111ea5ee16104ce6f0692d59244d7fdca32266bac1d9bbbe93a9941dcfbd0aL77-R79)) * Simplify `retry` function in `bicep.go` to use `DownloadToFolder` function and remove HTTP status code and response body logic ([link](https://github.com/radius-project/radius/pull/6426/files?diff=unified&w=0#diff-23111ea5ee16104ce6f0692d59244d7fdca32266bac1d9bbbe93a9941dcfbd0aL115-R113)) * Modify `TestGetDownloadURI` function in `binary_tools_test.go` to use new download URI format string and remove binary name parameter ([link](https://github.com/radius-project/radius/pull/6426/files?diff=unified&w=0#diff-6fd23df318670a45c0e8a3d45d308dcb80200d6e71536500003ef7bd6fa3020eL31-R36)) * Modify `GetDownloadURI` function in `binary_tools.go` to take only download URI format string as parameter and remove filename logic ([link](https://github.com/radius-project/radius/pull/6426/files?diff=unified&w=0#diff-0bb0df6a87062e6fad23c5f50750cb65e5e6e03112e526341f38a0e42ee06d6bL121-R128)) --- .github/workflows/functional-test.yaml | 6 ++ pkg/cli/bicep/bicep.go | 37 +---------- pkg/cli/tools/binary_tools.go | 80 ++++++++++++++++-------- pkg/cli/tools/binary_tools_test.go | 15 ----- test/functional/samples/tutorial_test.go | 2 +- 5 files changed, 65 insertions(+), 75 deletions(-) diff --git a/.github/workflows/functional-test.yaml b/.github/workflows/functional-test.yaml index 80e445be8d..858387dc2b 100644 --- a/.github/workflows/functional-test.yaml +++ b/.github/workflows/functional-test.yaml @@ -428,6 +428,12 @@ jobs: run: | helm repo add azure-workload-identity https://azure.github.io/azure-workload-identity/charts helm install workload-identity-webhook azure-workload-identity/workload-identity-webhook --namespace radius-default --create-namespace --version ${{ env.AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER }} --set azureTenantID=${{ secrets.INTEGRATION_TEST_TENANT_ID }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ env.GHCR_ACTOR }} + password: ${{ secrets.GH_RAD_CI_BOT_PAT }} - name: Download Bicep run: | chmod +x ./bin/rad diff --git a/pkg/cli/bicep/bicep.go b/pkg/cli/bicep/bicep.go index 11f7085f93..c32435b278 100644 --- a/pkg/cli/bicep/bicep.go +++ b/pkg/cli/bicep/bicep.go @@ -18,7 +18,6 @@ package bicep import ( "fmt" - "net/http" "os" "time" @@ -28,7 +27,6 @@ import ( const ( radBicepEnvVar = "RAD_BICEP" binaryName = "rad-bicep" - dirPrefix = "bicep-extensibility" retryAttempts = 10 retryDelaySecs = 5 ) @@ -74,15 +72,6 @@ func DeleteBicep() error { // DownloadBicep() attempts to download a file from a given URI and save it to a local filepath, retrying up to 10 times if // the download fails. If an error occurs, an error is returned. func DownloadBicep() error { - dirPrefix := "bicep-extensibility" - // Placeholders are for: channel, platform, filename - downloadURIFmt := fmt.Sprint("https://get.radapp.dev/tools/", dirPrefix, "/%s/%s/%s") - - uri, err := tools.GetDownloadURI(downloadURIFmt, binaryName) - if err != nil { - return err - } - filepath, err := tools.GetLocalFilepath(radBicepEnvVar, binaryName) if err != nil { return err @@ -90,7 +79,7 @@ func DownloadBicep() error { retryAttempts := 10 for attempt := 1; attempt <= retryAttempts; attempt++ { - success, err := retry(uri, filepath, attempt, retryAttempts) + success, err := retry(filepath, attempt, retryAttempts) if err != nil { return err } @@ -102,28 +91,8 @@ func DownloadBicep() error { return nil } -func retry(uri, filepath string, attempt, retryAttempts int) (bool, error) { - resp, err := http.Get(uri) - if err != nil { - if attempt == retryAttempts { - return false, fmt.Errorf("failed to download bicep: %v", err) - } - fmt.Printf("Attempt %d failed to download bicep: %v\nRetrying...", attempt, err) - time.Sleep(retryDelaySecs * time.Second) - return false, nil - } - defer resp.Body.Close() - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - if attempt == retryAttempts { - return false, fmt.Errorf("failed to download bicep from '%s' with status code: %d", uri, resp.StatusCode) - } - fmt.Printf("Attempt %d failed to download bicep from '%s' with status code: %d\nRetrying...", attempt, uri, resp.StatusCode) - time.Sleep(retryDelaySecs * time.Second) - return false, nil - } - - err = tools.DownloadToFolder(filepath, resp) +func retry(filepath string, attempt, retryAttempts int) (bool, error) { + err := tools.DownloadToFolder(filepath) if err != nil { if attempt == retryAttempts { return false, fmt.Errorf("failed to download bicep: %v", err) diff --git a/pkg/cli/tools/binary_tools.go b/pkg/cli/tools/binary_tools.go index 53dcf58d08..23bf896a51 100644 --- a/pkg/cli/tools/binary_tools.go +++ b/pkg/cli/tools/binary_tools.go @@ -17,14 +17,24 @@ limitations under the License. package tools import ( + "context" "fmt" - "io" - "net/http" "os" "path" "runtime" + credentials "github.com/oras-project/oras-credentials-go" "github.com/radius-project/radius/pkg/version" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + retry_lib "oras.land/oras-go/v2/registry/remote/retry" +) + +const ( + // binaryRepo is the name of the remote bicep binary repository + binaryRepo = "ghcr.io/radius-project/radius/bicep/rad-bicep/" ) // validPlatforms is a map of valid platforms to download for. The key is the combination of GOOS and GOARCH. @@ -116,52 +126,72 @@ func GetValidPlatform(currentOS, currentArch string) (string, error) { return platform, nil } -// GetDownloadURI takes in a download URI format string and a binary name, and returns a download URI -// string based on the runtime OS and architecture, or an error if the platform is not valid. -func GetDownloadURI(downloadURIFmt string, binaryName string) (string, error) { - filename, err := getFilename(binaryName) +// DownloadToFolder creates a folder and a file, uses the ORAS client to copy from the remote repository to the file, +// and makes the file executable by everyone. An error is returned if any of these steps fail. +func DownloadToFolder(filepath string) error { + // create folders + err := os.MkdirAll(path.Dir(filepath), os.ModePerm) if err != nil { - return "", err + return fmt.Errorf("failed to create folder %s: %v", path.Dir(filepath), err) + } + + // Create a file store + fs, err := file.New(path.Dir(filepath)) + if err != nil { + return fmt.Errorf("failed to create file store %s: %v", filepath, err) } + defer fs.Close() + ctx := context.Background() platform, err := GetValidPlatform(runtime.GOOS, runtime.GOARCH) if err != nil { - return "", err + return err } - return fmt.Sprintf(downloadURIFmt, version.Channel(), platform, filename), nil -} + // Define remote repository + repo, err := remote.NewRepository(binaryRepo + platform) + if err != nil { + return err + } -// DownloadToFolder creates a folder and a file, writes the response body to the file, and makes the file executable by -// everyone. An error is returned if any of these steps fail. -func DownloadToFolder(filepath string, resp *http.Response) error { - // create folders - err := os.MkdirAll(path.Dir(filepath), os.ModePerm) + // Create credentials to authenticate to repository + ds, err := credentials.NewStoreFromDocker(credentials.StoreOptions{ + AllowPlaintextPut: true, + }) if err != nil { - return fmt.Errorf("failed to create folder %s: %v", path.Dir(filepath), err) + return err } - // will truncate the file if it exists - out, err := os.Create(filepath) + repo.Client = &auth.Client{ + Client: retry_lib.DefaultClient, + Cache: auth.DefaultCache, + Credential: ds.Get, + } + + // Copy the artifact from the registry into the file store + tag := version.Channel() + if version.IsEdgeChannel() { + tag = "latest" + } + _, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) if err != nil { - return fmt.Errorf("failed to create file %s: %v", filepath, err) + return err } - defer out.Close() - // Write the body to file - _, err = io.Copy(out, resp.Body) + // Open the folder so we can mark it as executable + bicepBinary, err := os.Open(filepath) if err != nil { - return fmt.Errorf("failed to write file %s: %v", filepath, err) + return fmt.Errorf("failed to open file %s: %v", filepath, err) } // get the filemode so we can mark it as executable - file, err := out.Stat() + file, err := bicepBinary.Stat() if err != nil { return fmt.Errorf("failed to read file attributes %s: %v", filepath, err) } // make file executable by everyone - err = out.Chmod(file.Mode() | 0111) + err = bicepBinary.Chmod(file.Mode() | 0111) if err != nil { return fmt.Errorf("failed to change permissons for %s: %v", filepath, err) } diff --git a/pkg/cli/tools/binary_tools_test.go b/pkg/cli/tools/binary_tools_test.go index 683c4cdf60..b717b9f9bd 100644 --- a/pkg/cli/tools/binary_tools_test.go +++ b/pkg/cli/tools/binary_tools_test.go @@ -18,26 +18,11 @@ package tools import ( "errors" - "fmt" - "runtime" "testing" "github.com/stretchr/testify/require" - - "github.com/radius-project/radius/pkg/version" ) -func TestGetDownloadURI(t *testing.T) { - got, err := GetDownloadURI("%s/%s/%s", "test-bin") - require.NoError(t, err) - - platform, err := GetValidPlatform(runtime.GOOS, runtime.GOARCH) - require.NoError(t, err, "GetValidPlatform() error = %v", err) - want := fmt.Sprintf("%s/%s/test-bin", version.Channel(), platform) - - require.Equal(t, want, got, "GetDownloadURI() got = %v, want %v", got, want) -} - func TestGetValidPlatform(t *testing.T) { osArchTests := []struct { currentOS string diff --git a/test/functional/samples/tutorial_test.go b/test/functional/samples/tutorial_test.go index eee017a112..efd746e492 100644 --- a/test/functional/samples/tutorial_test.go +++ b/test/functional/samples/tutorial_test.go @@ -60,7 +60,7 @@ func Test_FirstApplicationSample(t *testing.T) { require.NoError(t, err) relPathSamplesRepo, err := filepath.Rel(cwd, samplesRepoAbsPath) require.NoError(t, err) - template := filepath.Join(relPathSamplesRepo, "demo/app.bicep") + template := filepath.Join(relPathSamplesRepo, "samples/demo/app.bicep") appName := "demo" appNamespace := "tutorial-demo"