From faebe07bdf8bab920fbecdef2426772179984932 Mon Sep 17 00:00:00 2001 From: Albert Cheng <38804567+ckairen@users.noreply.github.com> Date: Wed, 11 Dec 2024 09:08:22 -0800 Subject: [PATCH] Descend from alps (typespec apiview) (#31656) * typespec apiview & get changed filse * file checkpoint * revert change * temp save * initial * testing * telem aroudn impacted typespec * fix get impacted typespec * change swagger into typespec * resource provider * out null * new item force * telem artifacts dir * telem * telem * Fix file path for artifacts * Fix file path for artifacts * Fix file path for artifacts * Syntax * Syntax * shorten dir path * cleanup * readme variables rename * pr udpate: function name chanage + remove tspconfig as changed file * Update eng/scripts/Create-APIView.ps1 Co-authored-by: Mike Harder * add testing to contoso manager * targetting staging * adding scenario for new project without baseline * npm ci at earlier stage * syntax * fix baseline path * Add trailing newline * Update eng/pipelines/typespec-apiview.yml Co-authored-by: Ben Broderick Phillips * spelling * Update eng/scripts/Create-APIView.ps1 Co-authored-by: Wes Haggard * remove get ipmacted typespec * error check * add ignore core * spelling * try finally for git checkout * print out npx command * Update eng/pipelines/typespec-apiview.yml * comment * Update specification/contosowidgetmanager/Contoso.Management/employee.tsp * in case typespec folder is empty * test * remove test * create typeSpecAPIViewArtifactsDirectory * test * remove test * Update eng/scripts/Get-TypeSpec-Folders.ps1 * test * remove test * include all pr as trigger * test * remove test --------- Co-authored-by: Mike Harder Co-authored-by: Ben Broderick Phillips Co-authored-by: Wes Haggard --- eng/pipelines/typespec-apiview.yml | 54 +++++++++ eng/scripts/ChangedFiles-Functions.ps1 | 10 ++ eng/scripts/Create-APIView.ps1 | 152 +++++++++++++++++++++++-- eng/scripts/Get-TypeSpec-Folders.ps1 | 7 +- 4 files changed, 214 insertions(+), 9 deletions(-) create mode 100644 eng/pipelines/typespec-apiview.yml diff --git a/eng/pipelines/typespec-apiview.yml b/eng/pipelines/typespec-apiview.yml new file mode 100644 index 000000000000..53ee3e494aca --- /dev/null +++ b/eng/pipelines/typespec-apiview.yml @@ -0,0 +1,54 @@ +parameters: + - name: APIViewArtifactsDirectoryName + type: string + default: 'TypespecAPIViewArtifacts' + - name: APIViewArtifactsName + type: string + default: 'typeSpecAPIViewArtifacts' + - name: APIViewAPIUri + type: string + default: 'https://apiview.dev/PullRequest/DetectAPIChanges' + # Please use 'https://apiviewstagingtest.com/PullRequest/DetectAPIChanges' for testing purposes + +jobs: +- job: + pool: + name: azsdk-pool-mms-ubuntu-2204-general + vmImage: ubuntu-22.04 + + steps: + - checkout: self + fetchDepth: 0 + + - template: /eng/pipelines/templates/steps/npm-install.yml + + - pwsh: | + . $(Build.SourcesDirectory)/eng/scripts/Create-APIView.ps1 + New-TypeSpecAPIViewTokens ` + -TempDirectory "$(Agent.TempDirectory)" ` + -ArtifactsStagingDirectory "$(Build.ArtifactStagingDirectory)" ` + -APIViewArtifactsDirectoryName "${{ parameters.APIViewArtifactsDirectoryName }}" + displayName: Generate TypeSpec APIView Tokens + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/${{ parameters.APIViewArtifactsDirectoryName }}' + artifactName: '${{ parameters.APIViewArtifactsName }}' + publishLocation: 'pipeline' + displayName: 'Publish TypeSpec APIView Artifacts' + condition: and(succeeded(), ne(variables['Agent.JobStatus'], 'SucceededWithIssues')) + + - pwsh: | + . $(Build.SourcesDirectory)/eng/scripts/Create-APIView.ps1 + New-RestSpecsAPIViewReviews ` + -ArtiFactsStagingDirectory $(Build.ArtifactStagingDirectory) ` + -APIViewArtifactsDirectoryName ${{ parameters.APIViewArtifactsDirectoryName }} ` + -APIViewArtifactsName ${{ parameters.APIViewArtifactsName }} ` + -APIViewUri ${{ parameters.APIViewAPIUri }} ` + -BuildId $(Build.BuildId) ` + -RepoName $(Build.Repository.Name) ` + -PullRequestNumber $(System.PullRequest.PullRequestNumber)` + -Language 'TypeSpec' ` + -CommitSha $(Build.SourceVersion) + displayName: Create TypeSpec APIView + condition: and(succeeded(), ne(variables['Agent.JobStatus'], 'SucceededWithIssues')) diff --git a/eng/scripts/ChangedFiles-Functions.ps1 b/eng/scripts/ChangedFiles-Functions.ps1 index 927d4d3156a5..1a84809d4eb5 100644 --- a/eng/scripts/ChangedFiles-Functions.ps1 +++ b/eng/scripts/ChangedFiles-Functions.ps1 @@ -28,6 +28,16 @@ function Get-ChangedSwaggerFiles($changedFiles = (Get-ChangedFiles)) { return $changedSwaggerFiles } +function Get-ChangedTypeSpecFiles($changedFiles = (Get-ChangedFiles)) { + $changedFiles = Get-ChangedFilesUnderSpecification $changedFiles + + $changedTypeSpecFiles = $changedFiles.Where({ + $_.EndsWith(".tsp") + }) + + return $changedTypeSpecFiles +} + function Get-ChangedFilesUnderSpecification($changedFiles = (Get-ChangedFiles)) { $changedFilesUnderSpecification = $changedFiles.Where({ $_.StartsWith("specification") diff --git a/eng/scripts/Create-APIView.ps1 b/eng/scripts/Create-APIView.ps1 index 248f441e5805..987781886e45 100644 --- a/eng/scripts/Create-APIView.ps1 +++ b/eng/scripts/Create-APIView.ps1 @@ -166,6 +166,68 @@ function Invoke-SwaggerAPIViewParser { } } +<# +.DESCRIPTION + Invoke the TypeSpec parser to generate APIView tokens. + +.PARAMETER Type + New or Baseline TypeSpec APIView tokens. + +.PARAMETER ProjectPath + The TypeSpec Project path. + +.PARAMETER ResourceProvider + The ResourceProvider Name. + +.PARAMETER Tag + The Tag to use for generating the APIView Tokens. + +.PARAMETER TokenDirectory + The directory to store the generated APIView Tokens. + +.OUTPUTS + The resource provider name. +#> +function Invoke-TypeSpecAPIViewParser { + param ( + [ValidateSet("New", "Baseline")] + [Parameter(Mandatory = $true)] + [string]$Type, + [Parameter(Mandatory = $true)] + [string]$ProjectPath, + [Parameter(Mandatory = $true)] + [string]$ResourceProvider, + [Parameter(Mandatory = $true)] + [string]$TokenDirectory + ) + $tempWorkingDirectoryName = [guid]::NewGuid().ToString() + $tempWorkingDirectoryPath = [System.IO.Path]::Combine($TempDirectory, $tempWorkingDirectoryName) + New-Item -ItemType Directory -Path $tempWorkingDirectoryPath > $null + + try { + Write-Host "Compiling files and generating '$Type' APIView for '$resourceProvider'..." + Push-Location $ProjectPath + Write-Host "npm exec --no -- tsp compile . --emit=@azure-tools/typespec-apiview --option @azure-tools/typespec-apiview.emitter-output-dir=$tempWorkingDirectoryPath/output/apiview.json" + npm exec --no -- tsp compile . --emit=@azure-tools/typespec-apiview --option @azure-tools/typespec-apiview.emitter-output-dir=$tempWorkingDirectoryPath/output/apiview.json + if ($LASTEXITCODE) { + throw + } + Pop-Location + + $generatedAPIViewTokenFile = Get-ChildItem -File $tempWorkingDirectoryPath/output/apiview.json | Select-Object -First 1 + $apiViewTokensFilePath = [System.IO.Path]::Combine($TokenDirectory, "$resourceProvider.$Type.json") + Write-Host "Moving generated APIView Token file to '$apiViewTokensFilePath'" + Move-Item -Path $generatedAPIViewTokenFile.FullName -Destination $apiViewTokensFilePath -Force > $null + } catch { + LogError " Failed to generate '$Type' APIView Tokens on '$ProjectPath' for '$resourceProvider', please check the detail log and make sure TypeSpec compiler version is the latest." + throw + } finally { + if (Test-Path -Path $tempWorkingDirectoryPath) { + Remove-Item -Path $tempWorkingDirectoryPath -Recurse -Force > $null + } + } +} + <# .DESCRIPTION Generate New and Baseline APIView tokens for the changed swagger files in the PR. @@ -177,7 +239,7 @@ function Invoke-SwaggerAPIViewParser { .PARAMETER TempDirectory Temporary directory for files being processed. Use $(Agent.TempDirectory) on DevOps -.PARAMETER ArtiFactsStagingDirectory +.PARAMETER ArtifactsStagingDirectory The directory where the APIView tokens will be stored. Use $(Build.ArtifactStagingDirectory) on DevOps .PARAMETER APIViewArtifactsDirectoryName @@ -188,7 +250,7 @@ function New-SwaggerAPIViewTokens { [Parameter(Mandatory = $true)] [string]$TempDirectory, [Parameter(Mandatory = $true)] - [string]$ArtiFactsStagingDirectory, + [string]$ArtifactsStagingDirectory, [Parameter(Mandatory = $true)] [string]$APIViewArtifactsDirectoryName ) @@ -230,7 +292,7 @@ function New-SwaggerAPIViewTokens { $currentBranch = git rev-parse --abbrev-ref HEAD - $swaggerAPIViewArtifactsDirectory = [System.IO.Path]::Combine($ArtiFactsStagingDirectory, $APIViewArtifactsDirectoryName) + $swaggerAPIViewArtifactsDirectory = [System.IO.Path]::Combine($ArtifactsStagingDirectory, $APIViewArtifactsDirectoryName) # Generate Swagger APIView Tokens foreach ($entry in $autoRestConfigInfo.GetEnumerator()) { @@ -273,11 +335,87 @@ function New-SwaggerAPIViewTokens { LogGroupEnd } +<# +.DESCRIPTION + Generate New and Baseline APIView tokens for the changed TypeSpec files in the PR. + Detects the TypeSpec files changed in the PR and generates APIView tokens for the TypeSpec files. + New APIView tokens are generated using the default tag on the base branch. + Baseline APIView tokens are generated using the same tag on the target branch. + Script asumes that the merge commit is checked out. Such that Source commit = HEAD^ and Target commit = HEAD. + +.PARAMETER TempDirectory + Temporary directory for files being processed. Use $(Agent.TempDirectory) on DevOps + +.PARAMETER ArtifactsStagingDirectory + The directory where the APIView tokens will be stored. Use $(Build.ArtifactStagingDirectory) on DevOps + +.PARAMETER APIViewArtifactsDirectoryName + Name for the subdirectory where the APIView tokens will be stored. +#> +function New-TypeSpecAPIViewTokens { + param ( + [Parameter(Mandatory = $true)] + [string]$TempDirectory, + [Parameter(Mandatory = $true)] + [string]$ArtifactsStagingDirectory, + [Parameter(Mandatory = $true)] + [string]$APIViewArtifactsDirectoryName + ) + + $SourceCommitId = $(git rev-parse HEAD^) + $TargetCommitId = $(git rev-parse HEAD) + + $typeSpecProjects, $null = &"$PSScriptRoot/Get-TypeSpec-Folders.ps1" ` + -IgnoreCoreFiles:$true ` + -BaseCommitish:$SourceCommitId ` + -TargetCommitish:$TargetCommitId + + LogGroupStart " TypeSpec APIView Tokens will be generated for the following configuration files..." + $typeSpecProjects | ForEach-Object { + LogInfo " - $_" + } + LogGroupEnd + + $currentBranch = git rev-parse --abbrev-ref HEAD + + $typeSpecAPIViewArtifactsDirectory = [System.IO.Path]::Combine($ArtifactsStagingDirectory, $APIViewArtifactsDirectoryName) + New-Item -ItemType Directory -Path $typeSpecAPIViewArtifactsDirectory -Force | Out-Null + + try { + # Generate TypeSpec APIView Tokens + foreach ($typeSpecProject in $typeSpecProjects) { + $tokenDirectory = [System.IO.Path]::Combine($typeSpecAPIViewArtifactsDirectory, $typeSpecProject.split([IO.Path]::DirectorySeparatorChar)[-1]) + New-Item -ItemType Directory -Path $tokenDirectory -Force | Out-Null + + # Generate New APIView Token using default tag on base branch + git checkout $SourceCommitId + Invoke-TypeSpecAPIViewParser -Type "New" -ProjectPath $typeSpecProject -ResourceProvider $($typeSpecProject.split([IO.Path]::DirectorySeparatorChar)[-1]) -TokenDirectory $tokenDirectory + + # Generate BaseLine APIView Token using same tag on target branch + git checkout $TargetCommitId + + # Skip Baseline APIView Token for new projects + if (!(Test-Path -Path $typeSpecProject)) { + Write-Host "TypeSpec project $typeSpecProjectDir is not found in pull request target branch. API review will not have a baseline revision." + } + else { + Invoke-TypeSpecAPIViewParser -Type "Baseline" -ProjectPath $typeSpecProject -ResourceProvider $($typeSpecProject.split([IO.Path]::DirectorySeparatorChar)[-1]) -TokenDirectory $tokenDirectory | Out-Null + } + } + } + finally { + git checkout $currentBranch + LogGroupStart " See all generated TypeSpec APIView Artifacts..." + Get-ChildItem -Path $typeSpecAPIViewArtifactsDirectory -Recurse + LogGroupEnd + } +} + <# .DESCRIPTION Create APIView for the published packages. Send DevOps artifacts information to APIView to create APIView for the published packages. -.PARAMETER ArtiFactsStagingDirectory +.PARAMETER ArtifactsStagingDirectory The DevOps artifacts staging directory. Use $(Build.ArtifactStagingDirectory) on DevOps .PARAMETER APIViewArtifactsDirectoryName Temporary Directory for processing the APIView artifacts @@ -292,14 +430,14 @@ TGhe BuildId of the Run .PARAMETER PullRequestNumber The PR number .PARAMETER Language - The language of the resource provider `Swagger` + The language of the resource provider .PARAMETER CommitSha The commit sha of the current branch. Uusally the merge commit of the PR. #> function New-RestSpecsAPIViewReviews { param ( [Parameter(Mandatory = $true)] - [string]$ArtiFactsStagingDirectory, + [string]$ArtifactsStagingDirectory, [Parameter(Mandatory = $true)] [string]$APIViewArtifactsDirectoryName, [Parameter(Mandatory = $true)] @@ -318,7 +456,7 @@ function New-RestSpecsAPIViewReviews { [string]$CommitSha ) - $apiViewArtifactsDirectory = [System.IO.Path]::Combine($ArtiFactsStagingDirectory, $APIViewArtifactsDirectoryName) + $apiViewArtifactsDirectory = [System.IO.Path]::Combine($ArtifactsStagingDirectory, $APIViewArtifactsDirectoryName) $publishedPackages = Get-ChildItem -Path $apiViewArtifactsDirectory -Directory -ErrorAction SilentlyContinue Write-Host "Published packages: $publishedPackages" diff --git a/eng/scripts/Get-TypeSpec-Folders.ps1 b/eng/scripts/Get-TypeSpec-Folders.ps1 index e667216b0bdb..b7a51a69f169 100644 --- a/eng/scripts/Get-TypeSpec-Folders.ps1 +++ b/eng/scripts/Get-TypeSpec-Folders.ps1 @@ -1,5 +1,6 @@ [CmdletBinding()] param ( + [switch]$IgnoreCoreFiles = $false, [switch]$CheckAll = $false, [string]$BaseCommitish = "HEAD^", [string]$TargetCommitish = "HEAD" @@ -21,7 +22,7 @@ else { $changedFiles = @(Get-ChangedFiles -baseCommitish $BaseCommitish -targetCommitish $TargetCommitish -diffFilter "") $coreChangedFiles = Get-ChangedCoreFiles $changedFiles - if ($coreChangedFiles) { + if ($coreChangedFiles -and !$IgnoreCoreFiles) { Write-Verbose "Found changes to core eng or root files so checking all specs." $changedFiles = $checkAllPath $checkedAll = $true @@ -51,6 +52,8 @@ foreach ($skippedTypespecFolder in $skippedTypespecFolders | Select-Object -Uniq Write-Host "Cannot find directory $skippedTypespecFolder" } -$typespecFolders = $typespecFolders | ForEach-Object { [IO.Path]::GetRelativePath($repoPath, $_) -replace '\\', '/' } | Sort-Object -Unique +if ($typespecFolders.Length) { + $typespecFolders = $typespecFolders | ForEach-Object { [IO.Path]::GetRelativePath($repoPath, $_) -replace '\\', '/' } | Sort-Object -Unique +} return @($typespecFolders, $checkedAll)