From bcb396b3e6c9cc7c48051f6fa0192b65e5d34ad4 Mon Sep 17 00:00:00 2001 From: Judy Liu Date: Fri, 27 Dec 2024 11:13:49 +0800 Subject: [PATCH] automation support data plan (#23879) * tmp update * tmp update * tmp update * tmp update * update func * update func define * update func * update path * update referenced by comments * update referenced by comments * update referenced by comments * update referenced by comments * remove file * remove data plane logic filter * rename moduleRelativePath, relativePath all to packageModuleRelativePath --- .../cmd/v2/automation/automationCmd.go | 25 ++++++------------- .../cmd/v2/common/changelogProcessor.go | 7 +++--- .../generator/cmd/v2/common/constants.go | 3 +-- .../generator/cmd/v2/common/fileProcessor.go | 8 +++--- .../generator/cmd/v2/common/generation.go | 20 +++++++-------- .../generator/cmd/v2/release/releaseCmd.go | 2 +- eng/tools/generator/typespec/tspconfig.go | 8 ++++-- 7 files changed, 34 insertions(+), 39 deletions(-) diff --git a/eng/tools/generator/cmd/v2/automation/automationCmd.go b/eng/tools/generator/cmd/v2/automation/automationCmd.go index 41a657f2f8f4..6673d3c219d2 100644 --- a/eng/tools/generator/cmd/v2/automation/automationCmd.go +++ b/eng/tools/generator/cmd/v2/automation/automationCmd.go @@ -108,11 +108,6 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline // typespec // Generated by tsp only when tspconfig.yaml exists and has typespec-go option for _, tspProjectFolder := range input.RelatedTypeSpecProjectFolder { - if !strings.Contains(tspProjectFolder, ".Management") { - log.Printf("Skip processing: %s, as it is not a management plane project", tspProjectFolder) - continue - } - tspconfigPath := filepath.Join(input.SpecFolder, tspProjectFolder, "tspconfig.yaml") tsc, err := typespec.ParseTypeSpecConfig(tspconfigPath) if err != nil { @@ -136,14 +131,14 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline errorBuilder.add(err) continue } - + packageModuleRelativePath := tsc.GetPackageModuleRelativePath() namespaceResult, err := generateCtx.GenerateForTypeSpec(&common.GenerateParam{ RPName: module[0], NamespaceName: module[1], SkipGenerateExample: true, GoVersion: ctx.goVersion, TspClientOptions: []string{"--debug"}, - }) + }, packageModuleRelativePath) if err != nil { errorBuilder.add(err) continue @@ -152,8 +147,8 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline breaking := namespaceResult.Changelog.HasBreakingChanges() breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems() - srcFolder := filepath.Join(sdkRepo.Root(), "sdk", "resourcemanager", namespaceResult.RPName, namespaceResult.PackageName) - apiViewArtifact := filepath.Join(sdkRepo.Root(), "sdk", "resourcemanager", namespaceResult.RPName, namespaceResult.PackageName+".gosource") + srcFolder := filepath.Join(sdkRepo.Root(), packageModuleRelativePath) + apiViewArtifact := filepath.Join(sdkRepo.Root(), packageModuleRelativePath+".gosource") err := zipDirectory(srcFolder, apiViewArtifact) if err != nil { fmt.Println(err) @@ -161,16 +156,16 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline results = append(results, pipeline.PackageResult{ Version: namespaceResult.Version, - PackageName: fmt.Sprintf("sdk/resourcemanager/%s/%s", namespaceResult.RPName, namespaceResult.PackageName), - Path: []string{fmt.Sprintf("sdk/resourcemanager/%s/%s", namespaceResult.RPName, namespaceResult.PackageName)}, - PackageFolder: fmt.Sprintf("sdk/resourcemanager/%s/%s", namespaceResult.RPName, namespaceResult.PackageName), + PackageName: packageModuleRelativePath, + Path: []string{packageModuleRelativePath}, + PackageFolder: packageModuleRelativePath, TypespecProject: []string{tspProjectFolder}, Changelog: &pipeline.Changelog{ Content: &content, HasBreakingChange: &breaking, BreakingChangeItems: &breakingChangeItems, }, - APIViewArtifact: fmt.Sprintf("sdk/resourcemanager/%s/%s", namespaceResult.RPName, namespaceResult.PackageName+".gosource"), + APIViewArtifact: packageModuleRelativePath + ".gosource", Language: "Go", }) @@ -187,10 +182,6 @@ func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline } for _, readme := range input.RelatedReadmeMdFiles { - if !strings.Contains(readme, "resource-manager") { - log.Printf("Skip processing: %s, as it is not a management plane project", readme) - continue - } log.Printf("Start to process autorest project: %s", readme) diff --git a/eng/tools/generator/cmd/v2/common/changelogProcessor.go b/eng/tools/generator/cmd/v2/common/changelogProcessor.go index 1b1a944e39f5..b3bd46e78834 100644 --- a/eng/tools/generator/cmd/v2/common/changelogProcessor.go +++ b/eng/tools/generator/cmd/v2/common/changelogProcessor.go @@ -30,8 +30,9 @@ const ( sdk_remote_url = "https://github.com/Azure/azure-sdk-for-go.git" ) -func GetAllVersionTags(rpName, namespaceName string) ([]string, error) { - log.Printf("Fetching all release tags from GitHub for RP: '%s' Package: '%s' ...", rpName, namespaceName) +func GetAllVersionTags(packageModuleRelativePath string) ([]string, error) { + arr := strings.Split(packageModuleRelativePath, "/") + log.Printf("Fetching all release tags from GitHub for RP: '%s' Package: '%s' ...", arr[len(arr)-2], arr[len(arr)-1]) client := http.Client{} res, err := client.Get(sdk_tag_fetch_url) if err != nil { @@ -51,7 +52,7 @@ func GetAllVersionTags(rpName, namespaceName string) ([]string, error) { versionTag := make(map[string]string) for _, tag := range result { tagName := tag["ref"].(string) - if strings.Contains(tagName, "sdk/resourcemanager/"+rpName+"/"+namespaceName+"/v") { + if strings.Contains(tagName, packageModuleRelativePath+"/v") { m := regexp.MustCompile(semver.SemVerRegex).FindString(tagName) versions = append(versions, m) versionTag[m] = tagName diff --git a/eng/tools/generator/cmd/v2/common/constants.go b/eng/tools/generator/cmd/v2/common/constants.go index 2cd58144a3ec..836c12d3662b 100644 --- a/eng/tools/generator/cmd/v2/common/constants.go +++ b/eng/tools/generator/cmd/v2/common/constants.go @@ -4,6 +4,5 @@ package common const ( - ChangelogFileName = "CHANGELOG.md" - MgmtSDKModulePrefix = "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager" + ChangelogFileName = "CHANGELOG.md" ) diff --git a/eng/tools/generator/cmd/v2/common/fileProcessor.go b/eng/tools/generator/cmd/v2/common/fileProcessor.go index d6b2f110676c..5cfb5f692da1 100644 --- a/eng/tools/generator/cmd/v2/common/fileProcessor.go +++ b/eng/tools/generator/cmd/v2/common/fileProcessor.go @@ -416,7 +416,7 @@ func ReplaceNewClientNamePlaceholder(packageRootPath string, exports exports.Con return os.WriteFile(path, []byte(content), 0644) } -func UpdateModuleDefinition(packageRootPath, rpName, namespaceName string, version *semver.Version) error { +func UpdateModuleDefinition(packageRootPath, packageModuleRelativePath string, version *semver.Version) error { if version.Major() > 1 { path := filepath.Join(packageRootPath, "go.mod") @@ -431,7 +431,7 @@ func UpdateModuleDefinition(packageRootPath, rpName, namespaceName string, versi line = strings.TrimRight(line, "\r") parts := strings.Split(line, "/") if parts[len(parts)-1] != fmt.Sprintf("v%d", version.Major()) { - lines[i] = fmt.Sprintf("module github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/%s/%s/v%d", rpName, namespaceName, version.Major()) + lines[i] = fmt.Sprintf("module github.com/Azure/azure-sdk-for-go/%s/v%d", packageModuleRelativePath, version.Major()) } break } @@ -621,13 +621,13 @@ func existSuffixFile(path, suffix string) bool { return existed } -func replaceReadmeModule(path, rpName, namespaceName, currentVersion string) error { +func replaceReadmeModule(path, packageModuleRelativePath, currentVersion string) error { readmeFile, err := os.ReadFile(filepath.Join(path, "README.md")) if err != nil { return err } - module := fmt.Sprintf("github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/%s/%s", rpName, namespaceName) + module := fmt.Sprintf("github.com/Azure/azure-sdk-for-go/%s", packageModuleRelativePath) readmeModule := module match := regexp.MustCompile(fmt.Sprintf(`%s/v\d+`, module)) diff --git a/eng/tools/generator/cmd/v2/common/generation.go b/eng/tools/generator/cmd/v2/common/generation.go index 300dc724cc0d..3e229cb3d00e 100644 --- a/eng/tools/generator/cmd/v2/common/generation.go +++ b/eng/tools/generator/cmd/v2/common/generation.go @@ -213,7 +213,7 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate if !onBoard { log.Printf("Get ori exports for changelog generation...") - tags, err := GetAllVersionTags(generateParam.RPName, generateParam.NamespaceName) + tags, err := GetAllVersionTags(fmt.Sprintf("sdk/resourcemanager/%s/%s", generateParam.RPName, generateParam.NamespaceName)) if err != nil { return nil, err } @@ -308,7 +308,7 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate } log.Printf("Update module definition if v2+...") - err = UpdateModuleDefinition(packagePath, generateParam.RPName, generateParam.NamespaceName, version) + err = UpdateModuleDefinition(packagePath, fmt.Sprintf("sdk/resourcemanager/%s/%s", generateParam.RPName, generateParam.NamespaceName), version) if err != nil { return nil, err } @@ -341,7 +341,7 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate } log.Printf("Replace README.md module...") - if err = replaceReadmeModule(packagePath, generateParam.RPName, generateParam.NamespaceName, version.String()); err != nil { + if err = replaceReadmeModule(packagePath, fmt.Sprintf("sdk/resourcemanager/%s/%s", generateParam.RPName, generateParam.NamespaceName), version.String()); err != nil { return nil, err } @@ -374,8 +374,8 @@ func (ctx *GenerateContext) GenerateForSingleRPNamespace(generateParam *Generate } } -func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (*GenerateResult, error) { - packagePath := filepath.Join(ctx.SDKPath, "sdk", "resourcemanager", generateParam.RPName, generateParam.NamespaceName) +func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam, packageModuleRelativePath string) (*GenerateResult, error) { + packagePath := filepath.Join(ctx.SDKPath, packageModuleRelativePath) changelogPath := filepath.Join(packagePath, ChangelogFileName) version, err := semver.NewVersion("0.1.0") @@ -448,13 +448,13 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (* if !onBoard { log.Printf("Get ori exports for changelog generation...") - tags, err := GetAllVersionTags(generateParam.RPName, generateParam.NamespaceName) + tags, err := GetAllVersionTags(packageModuleRelativePath) if err != nil { return nil, err } if len(tags) == 0 { - return nil, fmt.Errorf("github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/%s/%s hasn't been released, it's supposed to OnBoard", generateParam.RPName, generateParam.NamespaceName) + return nil, fmt.Errorf("github.com/Azure/azure-sdk-for-go/%s hasn't been released, it's supposed to OnBoard", packageModuleRelativePath) } previousVersionTag := GetPreviousVersionTag(isCurrentPreview, tags) @@ -546,7 +546,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (* } log.Printf("Update module definition if v2+...") - err = UpdateModuleDefinition(packagePath, generateParam.RPName, generateParam.NamespaceName, version) + err = UpdateModuleDefinition(packagePath, packageModuleRelativePath, version) if err != nil { return nil, err } @@ -561,7 +561,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (* return nil, err } - baseModule := fmt.Sprintf("%s/%s/%s", MgmtSDKModulePrefix, generateParam.RPName, generateParam.NamespaceName) + baseModule := fmt.Sprintf("%s/%s", "github.com/Azure/azure-sdk-for-go", packageModuleRelativePath) if _, err := os.Stat(filepath.Join(packagePath, "fake")); !os.IsNotExist(err) && oldModuleVersion.Major() != version.Major() { log.Printf("Replace fake module v2+...") if err = ReplaceModule(version, packagePath, baseModule, ".go"); err != nil { @@ -578,7 +578,7 @@ func (ctx *GenerateContext) GenerateForTypeSpec(generateParam *GenerateParam) (* } log.Printf("Replace README.md module...") - if err = replaceReadmeModule(packagePath, generateParam.RPName, generateParam.NamespaceName, version.String()); err != nil { + if err = replaceReadmeModule(packagePath, packageModuleRelativePath, version.String()); err != nil { return nil, err } diff --git a/eng/tools/generator/cmd/v2/release/releaseCmd.go b/eng/tools/generator/cmd/v2/release/releaseCmd.go index 42e0a866c223..5b9187a7efb1 100644 --- a/eng/tools/generator/cmd/v2/release/releaseCmd.go +++ b/eng/tools/generator/cmd/v2/release/releaseCmd.go @@ -194,7 +194,7 @@ func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash str GoVersion: c.flags.GoVersion, TypeSpecEmitOption: c.flags.TypeSpecGoOption, TspClientOptions: c.flags.TspClientOption, - }) + }, generateCtx.TypeSpecConfig.GetPackageModuleRelativePath()) } else { log.Printf("Generate SDK through AutoRest...") result, err = generateCtx.GenerateForSingleRPNamespace(&common.GenerateParam{ diff --git a/eng/tools/generator/typespec/tspconfig.go b/eng/tools/generator/typespec/tspconfig.go index 662999e346f0..5cc6234757a4 100644 --- a/eng/tools/generator/typespec/tspconfig.go +++ b/eng/tools/generator/typespec/tspconfig.go @@ -117,6 +117,10 @@ func ParseTypeSpecConfig(tspconfigPath string) (*TypeSpecConfig, error) { return &tspConfig, err } +func (tc *TypeSpecConfig) GetPackageModuleRelativePath() string { + return tc.Options["@azure-tools/typespec-go"].(map[string]interface{})["service-dir"].(string) + "/" + tc.Options["@azure-tools/typespec-go"].(map[string]interface{})["package-dir"].(string) +} + func (tc *TypeSpecConfig) EditOptions(emit string, option map[string]any, append bool) { if tc.Options == nil { tc.Options = make(map[string]any) @@ -177,8 +181,8 @@ func (tc TypeSpecConfig) GetModuleName() ([2]string, error) { if l != 7 { return [2]string{}, fmt.Errorf("module is invalid and must be in the format of `github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/{rpName}/{packageName}`") } - if !strings.Contains(s[l-1], "arm") { - return [2]string{}, fmt.Errorf("packageName is invalid and must start with `arm`") + if !strings.Contains(s[l-1], "arm") && !strings.Contains(s[l-1], "az") { + return [2]string{}, fmt.Errorf("packageName is invalid and must start with `arm` or `az`") } return [2]string{s[l-2], s[l-1]}, nil