diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 2549f6b18..8309e5ffe 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -19,7 +19,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent $defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" -$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName") +$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","EnvironmentName") $runAlPipelineOverrides = @( "DockerPull" @@ -38,6 +38,8 @@ $runAlPipelineOverrides = @( "InstallMissingDependencies" "PreCompileApp" "PostCompileApp" + "PipelineInitialize" + "PipelineFinalize" ) # Well known AppIds @@ -476,6 +478,10 @@ function MergeCustomObjectIntoOrderedDictionary { if ($srcPropType -eq "PSCustomObject" -and $dstPropType -eq "OrderedDictionary") { MergeCustomObjectIntoOrderedDictionary -dst $dst."$prop" -src $srcProp } + elseif ($dstProp -is [String] -and $srcProp -is [Object[]]) { + # For properties like "runs-on", which is a string, you can specify an array in settings, which gets joined with a comma + $dst."$prop" = $srcProp -join ', ' + } elseif ($dstPropType -ne $srcPropType -and !($srcPropType -eq "Int64" -and $dstPropType -eq "Int32")) { # Under Linux, the Int fields read from the .json file will be Int64, while the settings defaults will be Int32 # This is not seen as an error and will not throw an error @@ -565,6 +571,7 @@ function ReadSettings { $settings = [ordered]@{ "type" = "PTE" "unusedALGoSystemFiles" = @() + "customALGoSystemFiles" = @() "projects" = @() "powerPlatformSolutionFolder" = "" "country" = "us" diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 766d7ee8b..245a006ab 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -1,3 +1,17 @@ +function GetCustomizationAnchors { + return @{ + "_BuildALGoProject.yaml" = @{ + "BuildALGoProject" = @( + @{ "Step" = 'Read settings'; "Before" = $false } + @{ "Step" = 'Read secrets'; "Before" = $false } + @{ "Step" = 'Build'; "Before" = $true } + @{ "Step" = 'Build'; "Before" = $false } + @{ "Step" = 'Cleanup'; "Before" = $true } + ) + } + } +} + <# .SYNOPSIS Downloads a template repository and returns the path to the downloaded folder @@ -397,7 +411,7 @@ function IsDirectALGo { function GetSrcFolder { Param( - [hashtable] $repoSettings, + [string] $repoType, [string] $templateUrl, [string] $templateFolder, [string] $srcPath @@ -409,7 +423,7 @@ function GetSrcFolder { return '' } if (IsDirectALGo -templateUrl $templateUrl) { - switch ($repoSettings.type) { + switch ($repoType) { "PTE" { $typePath = "Per Tenant Extension" } @@ -440,32 +454,153 @@ function UpdateSettingsFile { Param( [string] $settingsFile, [hashtable] $updateSettings, - [hashtable] $additionalSettings = @{} + [hashtable] $indirectTemplateSettings = @{} ) + $modified = $false # Update Repo Settings file with the template URL if (Test-Path $settingsFile) { $settings = Get-Content $settingsFile -Encoding UTF8 | ConvertFrom-Json } else { $settings = [PSCustomObject]@{} + $modified = $true } foreach($key in $updateSettings.Keys) { if ($settings.PSObject.Properties.Name -eq $key) { - $settings."$key" = $updateSettings."$key" + if ($settings."$key" -ne $updateSettings."$key") { + $settings."$key" = $updateSettings."$key" + $modified = $true + } } else { # Add the property if it doesn't exist $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $updateSettings."$key" + $modified = $true } } - # Grab settings from additionalSettings if they are not already in settings - foreach($key in $additionalSettings.Keys) { + # Grab settings from indirectTemplateSettings if they are not already in settings + foreach($key in $indirectTemplateSettings.Keys) { + # CustomALGoSystemFiles will not be copied from the indirect template settings - they will be applied to the indirect template + # UnusedALGoSystemFiles will not be copied from the indirect template settings - they will be used during the update process + if (@('customALGoSystemFiles','unusedALGoSystemFiles') -contains $key) { + continue + } if (!($settings.PSObject.Properties.Name -eq $key)) { # Add the property if it doesn't exist - $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $additionalSettings."$key" + $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $indirectTemplateSettings."$key" + $modified = $true + } + } + + if ($modified) { + # Save the file with LF line endings and UTF8 encoding + $settings | Set-JsonContentLF -path $settingsFile + } + return $modified +} + +function GetCustomALGoSystemFiles { + Param( + [string] $baseFolder, + [hashtable] $settings, + [string[]] $projects + ) + + function YieldItem{ + Param( + [string] $baseFolder, + [string] $source, + [string] $destination, + [string[]] $projects + ) + + if ($destination -like ".AL-Go$([IO.Path]::DirectorySeparatorChar)*") { + $destinations = $projects | ForEach-Object { Join-Path $_ $destination } + } + else { + $destinations = @($destination) + } + $destinations | ForEach-Object { + Write-Host "- $_" + $content = Get-ContentLF -Path $source + $existingFile = Join-Path $baseFolder $_ + $existingContent = '' + if (Test-Path -Path $existingFile) { + $existingContent = Get-ContentLF -Path $existingFile + } + if ($content -ne $existingContent) { + Write-Output @{ "DstFile" = $_; "content" = $content } + } + } + } + + if ($settings.customALGoSystemFiles -isnot [Array]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + foreach($customspec in $settings.customALGoSystemFiles) { + if ($customspec -isnot [Hashtable]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (!($customSpec.ContainsKey('Source') -and $customSpec.ContainsKey('Destination'))) { + throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be specified. See https://aka.ms/algosettings#customalgosystemfiles." + } + $source = $customspec.Source + $destination = $customSpec.Destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) + if ($destination -isnot [string] -or $destination -eq '') { + throw "customALGoSystemFiles setting is wrongly formatted, Destination must be a string, which isn't blank. See https://aka.ms/algosettings#customalgosystemfiles." + } + if ($source -isnot [string] -or $source -notlike 'https://*' -or (-not [System.Uri]::IsWellFormedUriString($source,1))) { + throw "customALGoSystemFiles setting is wrongly formatted, Source must secure download URL. See https://aka.ms/algosettings#customalgosystemfiles." + } + + $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + New-Item -Path $tempFolder -ItemType Directory | Out-Null + $ext = [System.IO.Path]::GetExtension($source) + $zipName = "$tempFolder$ext" + try { + if ($ext -eq '.zip') { + Write-Host "$($destination):" + if ($customSpec.ContainsKey('FileSpec')) { $fileSpec = $customSpec.FileSpec } else { $fileSpec = '*' } + if ($customSpec.ContainsKey('Recurse')) { $recurse = $customSpec.Recurse } else { $recurse = $true } + if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { + throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (!($destination.EndsWith('/') -or $destination.EndsWith('\'))) { + throw "customALGoSystemFiles setting is wrongly formatted, destination must be a folder (terminated with / or \). See https://aka.ms/algosettings#customalgosystemfiles." + } + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName + Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force + $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve + Push-Location -Path $subFolder + try { + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + $destRelativeFileName = Resolve-Path $_.FullName -Relative + $destFileName = Join-Path $destination $destRelativeFileName + $destFileName = $destFileName.TrimStart('\/') + YieldItem -baseFolder $baseFolder -source $_.FullName -destination $destFileName -projects $projects + } + } + finally { + Pop-Location + } + } + else { + if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { + throw "customALGoSystemFiles setting is wrongly formatted, FileSpec and Recurse are only allowed with .zip files. See https://aka.ms/algosettings#customalgosystemfiles." + } + if ($destination.endsWith([IO.Path]::DirectorySeparatorChar)) { + $destination = Join-Path $destination ([System.IO.Path]::GetFileName($source)) + } + Write-Host "$($destination):" + $tempFilename = Join-Path $tempFolder ([System.IO.Path]::GetFileName($source)) + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $tempFilename + YieldItem -baseFolder $baseFolder -source $tempFilename -destination $destination -projects $projects + } + } + finally { + if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } + Remove-Item -Path $tempFolder -Recurse -Force } } - # Save the file with LF line endings and UTF8 encoding - $settings | Set-JsonContentLF -path $settingsFile } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 405b02064..db9b988cd 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -1,8 +1,10 @@ Param( [Parameter(HelpMessage = "The GitHub actor running the action", Mandatory = $false)] [string] $actor, - [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] + [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] [string] $token, + [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] + [string] $githubToken, [Parameter(HelpMessage = "URL of the template repository (default is the template repository used to create the repository)", Mandatory = $false)] [string] $templateUrl = "", [Parameter(HelpMessage = "Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update)", Mandatory = $true)] @@ -22,12 +24,18 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper -if ($update -eq 'Y') { - if (-not $token) { +if ($token) { + # Specified token is GhTokenWorkflow secret - decode from base 64 + Write-Host "Using ghTokenWorkflow secret" + $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) +} +else { + if ($update -eq 'Y') { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." } else { - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) + Write-Host "Using GitHub Token" + $token = $githubToken } } @@ -69,6 +77,7 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { $downloadLatest = $true } +$realTemplateFolder = $null $templateFolder = DownloadTemplateRepository -headers $headers -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" @@ -76,13 +85,45 @@ $templateBranch = $templateUrl.Split('@')[1] $templateOwner = $templateUrl.Split('/')[3] $templateInfo = "$templateOwner/$($templateUrl.Split('/')[4])" +$indirectTemplateRepoSettings = @{} +$indirectTemplateProjectSettings = @{} + $isDirectALGo = IsDirectALGo -templateUrl $templateUrl if (-not $isDirectALGo) { - $ALGoSettingsFile = Join-Path $templateFolder "*/$repoSettingsFile" - if (Test-Path -Path $ALGoSettingsFile -PathType Leaf) { - $templateRepoSettings = Get-Content $ALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + $myRepoSettingsFile = Join-Path $templateFolder "*/$RepoSettingsFile" + if (Test-Path -Path $myRepoSettingsFile -PathType Leaf) { + $templateRepoSettings = Get-Content $myRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { - throw "The specified template repository is not a template repository, but instead another AL-Go repository. This is not supported." + # The template repository is a url to another AL-Go repository (an indirect template repository) + # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the indirect template reposotiry points to the "real" template repository + # Copy files and folders from the indirect template repository, but grab the unmodified file from the "real" template repository if it exists and apply customizations + Write-Host "Indirect AL-Go template repository detected, downloading the 'real' template repository" + $realTemplateUrl = $templateRepoSettings.templateUrl + if ($templateRepoSettings.Keys -contains "templateSha") { + $realTemplateSha = $templateRepoSettings.templateSha + } + else { + $realTemplateSha = "" + } + # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository + $realTemplateFolder = DownloadTemplateRepository -headers $headers -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') + Write-Host "Real Template Folder: $realTemplateFolder" + + # Set TemplateBranch and TemplateOwner + # Keep TemplateUrl and TemplateSha pointing to the indirect template repository + $templateBranch = $realTemplateUrl.Split('@')[1] + $templateOwner = $realTemplateUrl.Split('/')[3] + + $indirectTemplateRepoSettings = $templateRepoSettings + # If the indirect template contains unusedALGoSystemFiles, we need to remove them from the current repository + if ($indirectTemplateRepoSettings.ContainsKey('unusedALGoSystemFiles')) { + $unusedALGoSystemFiles += $indirectTemplateRepoSettings.unusedALGoSystemFiles + } + $myALGoSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" + if (Test-Path $myALGoSettingsFile -PathType Leaf) { + Write-Host "Read project settings from indirect template repository" + $indirectTemplateProjectSettings = Get-Content $myALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + } } } } @@ -133,10 +174,31 @@ foreach($checkfile in $checkfiles) { $srcPath = $checkfile.srcPath $dstPath = $checkfile.dstPath $dstFolder = Join-Path $baseFolder $dstPath - $srcFolder = GetSrcFolder -repoSettings $repoSettings -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath + $srcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath + $realSrcFolder = $null + if ($realTemplateFolder) { + $realSrcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath + } if ($srcFolder) { Push-Location -Path $srcFolder try { + if ($srcPath -eq '.AL-Go' -and $type -eq "script" -and $realSrcFolder) { + Write-Host "Update Project Settings" + # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) + $projectSettingsFile = Join-Path $dstFolder "settings.json" + if (UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -indirectTemplateSettings $indirectTemplateProjectSettings) { + $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } + } + } + + # Remove unused AL-Go system files + $unusedALGoSystemFiles | ForEach-Object { + if (Test-Path -Path (Join-Path $dstFolder $_) -PathType Leaf) { + Write-Host "Remove unused AL-Go system file: $_" + $removeFiles += @(Join-Path $dstPath $_) + } + } + # Loop through all files in the template repository matching the pattern Get-ChildItem -Path $srcFolder -Filter $checkfile.pattern | ForEach-Object { # Read the template file and modify it based on the settings @@ -145,45 +207,61 @@ foreach($checkfile in $checkfiles) { Write-Host "- $filename" $dstFile = Join-Path $dstFolder $fileName $srcFile = $_.FullName + $realSrcFile = $srcFile + $isFileDirectALGo = $isDirectALGo Write-Host "SrcFolder: $srcFolder" + if ($realSrcFolder) { + # if SrcFile is an indirect template repository, we need to find the file in the "real" template repository + $fname = Join-Path $realSrcFolder (Resolve-Path $srcFile -Relative) + if (Test-Path -Path $fname -PathType Leaf) { + Write-Host "File is available in the 'real' template repository" + $realSrcFile = $fname + $isFileDirectALGo = IsDirectALGo -templateUrl $realTemplateUrl + } + } if ($type -eq "workflow") { # for workflow files, we might need to modify the file based on the settings - $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $srcFile -repoSettings $repoSettings -depth $depth -includeBuildPP $includeBuildPP + $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $realsrcFile -repoSettings $repoSettings -depth $depth -includeBuildPP $includeBuildPP } else { # For non-workflow files, just read the file content - $srcContent = Get-ContentLF -Path $srcFile + $srcContent = Get-ContentLF -Path $realSrcFile } # Replace static placeholders $srcContent = $srcContent.Replace('{TEMPLATEURL}', $templateUrl) - if ($isDirectALGo) { + if ($isFileDirectALGo) { # If we are using direct AL-Go repo, we need to change the owner to the remplateOwner, the repo names to AL-Go and AL-Go/Actions and the branch to templateBranch ReplaceOwnerRepoAndBranch -srcContent ([ref]$srcContent) -templateOwner $templateOwner -templateBranch $templateBranch } - $dstFileExists = Test-Path -Path $dstFile -PathType Leaf - if ($unusedALGoSystemFiles -contains $fileName) { - # file is not used by ALGo, remove it if it exists - # do not add it to $updateFiles if it does not exist - if ($dstFileExists) { - $removeFiles += @(Join-Path $dstPath $filename) - } + $customizationAnchors = GetCustomizationAnchors + if ($type -eq 'workflow' -and $realSrcFile -ne $srcFile) { + # Apply customizations from indirect template repository + Write-Host "Apply customizations from indirect template repository: $srcFile" + [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $customizationAnchors) } - elseif ($dstFileExists) { - # file exists, compare and add to $updateFiles if different - $dstContent = Get-ContentLF -Path $dstFile - if ($dstContent -cne $srcContent) { - Write-Host "Updated $type ($(Join-Path $dstPath $filename)) available" + + if ($unusedALGoSystemFiles -notcontains $fileName) { + if (Test-Path -Path $dstFile -PathType Leaf) { + if ($type -eq 'workflow') { + Write-Host "Apply customizations from my repository: $dstFile" + [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $customizationAnchors) + } + # file exists, compare and add to $updateFiles if different + $dstContent = Get-ContentLF -Path $dstFile + if ($dstContent -cne $srcContent) { + Write-Host "Updated $type ($(Join-Path $dstPath $filename)) available" + $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } + } + } + else { + # new file, add to $updateFiles + Write-Host "New $type ($(Join-Path $dstPath $filename)) available" $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } } } - else { - # new file, add to $updateFiles - Write-Host "New $type ($(Join-Path $dstPath $filename)) available" - $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } - } } } finally { @@ -192,9 +270,21 @@ foreach($checkfile in $checkfiles) { } } +# Apply Custom AL-Go System Files from settings +$updateFiles += @(GetCustomALGoSystemFiles -baseFolder $baseFolder -settings $repoSettings -projects $projects) + if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available if (($updateFiles) -or ($removeFiles)) { + if ($updateFiles) { + Write-Host "Updated files:" + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" } + + } + if ($removeFiles) { + Write-Host "Removed files:" + $removeFiles | ForEach-Object { Write-Host "- $_" } + } OutputWarning -message "There are updates for your AL-Go system, run 'Update AL-Go System Files' workflow to download the latest version of AL-Go." } else { @@ -218,7 +308,10 @@ else { invoke-git status - UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } + $repoSettingsFile = Join-Path ".github" "AL-Go-Settings.json" + if (UpdateSettingsFile -settingsFile $repoSettingsFile -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -indirectTemplateSettings $indirectTemplateRepoSettings) { + $updateFiles += @{ "DstFile" = $repoSettingsFile; "content" = (Get-Content -Path $repoSettingsFile -Encoding UTF8 -Raw) } + } # Update the files # Calculate the release notes, while updating diff --git a/Actions/CheckForUpdates/README.md b/Actions/CheckForUpdates/README.md index 554ca9db5..c3db39db0 100644 --- a/Actions/CheckForUpdates/README.md +++ b/Actions/CheckForUpdates/README.md @@ -14,7 +14,8 @@ none | :-- | :-: | :-- | :-- | | shell | | The shell (powershell or pwsh) in which the PowerShell script in this action should run | powershell | | actor | | The GitHub actor running the action | github.actor | -| token | | The GitHub token running the action | github.token | +| token | | Base64 encoded GhTokenWorkflow secret | | +| githubToken | | The GitHub token running the action | github.token | | templateUrl | | URL of the template repository (default is the template repository used to create the repository) | default | | downloadLatest | Yes | Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update) | | | update | | Set this input to Y in order to update AL-Go System Files if needed | N | diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index 4b12bc6f7..b1dafdad6 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -14,6 +14,10 @@ inputs: required: false default: ${{ github.actor }} token: + description: Base64 encoded GhTokenWorkflow secret + required: false + default: '' + githubToken: description: The GitHub token running the action required: false default: ${{ github.token }} @@ -44,6 +48,7 @@ runs: env: _actor: ${{ inputs.actor }} _token: ${{ inputs.token }} + _githubToken: ${{ inputs.githubToken }} _templateUrl: ${{ inputs.templateUrl }} _downloadLatest: ${{ inputs.downloadLatest }} _update: ${{ inputs.update }} @@ -51,7 +56,7 @@ runs: _directCommit: ${{ inputs.directCommit }} run: | ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "CheckForUpdates" -Action { - ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') + ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -githubToken $ENV:_githubToken -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') } branding: icon: terminal diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 5b231aea7..a109ed574 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -20,7 +20,7 @@ class Yaml { # Save the Yaml file with LF line endings using UTF8 encoding Save([string] $filename) { - $this.content | Set-ContentLF -Path $filename + $this.content -join "`n" | Set-ContentLF -Path $filename } # Find the lines for the specified Yaml path, given by $line @@ -126,6 +126,45 @@ class Yaml { return $this.Get($line, [ref] $start, [ref] $count) } + # Locate all lines in the next level of a yaml path + # if $line is empty, you get all first level lines + # Example: + # GetNextLevel("jobs:/") returns @("Initialization:","CheckForUpdates:","Build:","Deploy:",...) + [string[]] GetNextLevel([string] $line) { + [int]$start = 0 + [int]$count = 0 + [Yaml] $yaml = $this + if ($line) { + $yaml = $this.Get($line, [ref] $start, [ref] $count) + } + return $yaml.content | Where-Object { $_ -and -not $_.StartsWith(' ') } + } + + # Get the value of a property as a string + # Example: + # GetProperty("jobs:/Build:/needs:") returns "[ Initialization, Build1 ]" + [string] GetProperty([string] $line) { + [int]$start = 0 + [int]$count = 0 + [Yaml] $yaml = $this.Get($line, [ref] $start, [ref] $count) + if ($yaml -and $yaml.content.Count -eq 1) { + return $yaml.content[0].SubString($yaml.content[0].IndexOf(':')+1).Trim() + } + return $null + } + + # Get the value of a property as a string array + # Example: + # GetPropertyArray("jobs:/Build:/needs:") returns @("Initialization", "Build") + [string[]] GetPropertyArray([string] $line) { + $prop = $this.GetProperty($line) + if ($prop) { + # "needs: [ Initialization, Build ]" becomes @("Initialization", "Build") + return $prop.TrimStart('[').TrimEnd(']').Split(',').Trim() + } + return $null + } + # Replace the lines for the specified Yaml path, given by $line with the lines in $content # If $line ends with '/', then the lines for the section are replaced only # If $line doesn't end with '/', then the line + the lines for the section are replaced @@ -185,4 +224,255 @@ class Yaml { $this.content = $this.content[0..($index-1)] + $yamlContent + $this.content[$index..($this.content.Count-1)] } } + + # Add lines to Yaml content + [void] Add([string[]] $yamlContent) { + if (!$yamlContent) { + return + } + $this.Insert($this.content.Count, $yamlContent) + } + + # Locate jobs in YAML based on a name pattern + # Example: + # GetCustomJobsFromYaml() returns @("CustomJob1", "CustomJob2") + # GetCustomJobsFromYaml("Build*") returns @("Build1","Build2","Build") + [hashtable[]] GetCustomJobsFromYaml([string] $name) { + $result = @() + $allJobs = $this.GetNextLevel('jobs:/').Trim(':') + $customJobs = @($allJobs | Where-Object { $_ -like $name }) + if ($customJobs) { + $nativeJobs = @($allJobs | Where-Object { $customJobs -notcontains $_ }) + Write-Host "Native Jobs:" + foreach($nativeJob in $nativeJobs) { + Write-Host "- $nativeJob" + } + Write-Host "Custom Jobs:" + foreach($customJob in $customJobs) { + Write-Host "- $customJob" + $jobsWithDependency = $nativeJobs | Where-Object { $this.GetPropertyArray("jobs:/$($_):/needs:") | Where-Object { $_ -eq $customJob } } + # If any Build Job has a dependency on this CustomJob, add will be added to all build jobs later + if ($jobsWithDependency | Where-Object { $_ -like 'Build*' }) { + $jobsWithDependency = @($jobsWithDependency | Where-Object { $_ -notlike 'Build*' }) + @('Build') + } + if ($jobsWithDependency) { + Write-Host " - Jobs with dependency: $($jobsWithDependency -join ', ')" + $result += @(@{ "Name" = $customJob; "Content" = @($this.Get("jobs:/$($customJob):").content); "NeedsThis" = @($jobsWithDependency) }) + } + } + } + return $result + } + + # Add jobs to Yaml and update Needs section from native jobs which needs this custom Job + # $customJobs is an array of hashtables with Name, Content and NeedsThis + # Example: + # $customJobs = @(@{ "Name" = "CustomJob1"; "Content" = @(" - pwsh"," -File Build1"); "NeedsThis" = @("Initialization", "Build") }) + # AddCustomJobsToYaml($customJobs) + # The function will add the job CustomJob1 to the Yaml file and update the Needs section of Initialization and Build + # The function will not add the job CustomJob1 if it already exists + [void] AddCustomJobsToYaml([hashtable[]] $customJobs) { + $existingJobs = $this.GetNextLevel('jobs:/').Trim(':') + Write-Host "Adding New Jobs" + foreach($customJob in $customJobs) { + if ($existingJobs -contains $customJob.Name) { + Write-Host "Job $($customJob.Name) already exists" + continue + } + Write-Host "$($customJob.Name) has dependencies from $($customJob.NeedsThis -join ',')" + foreach($needsthis in $customJob.NeedsThis) { + if ($needsthis -eq 'Build') { + $existingJobs | Where-Object { $_ -like 'Build*'} | ForEach-Object { + # Add dependency to all build jobs + $this.Replace("jobs:/$($_):/needs:","needs: [ $(@($this.GetPropertyArray("jobs:/$($_):/needs:"))+@($customJob.Name) -join ', ') ]") + } + } + elseif ($existingJobs -contains $needsthis) { + # Add dependency to job + $needs = @(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) | Where-Object { $_ }) -join ', ' + $this.Replace("jobs:/$($needsthis):/needs:","needs: [ $needs ]") + } + } + $this.content += @('') + @($customJob.content | ForEach-Object { " $_" }) + } + } + + [string[]] GetStepsFromJob([string] $job) { + $steps = $this.GetNextLevel("Jobs:/$($job):/steps:/") | Where-Object { $_ -like '- name: *' } | ForEach-Object { $_.Substring(8).Trim() } + if ($steps | Group-Object | Where-Object { $_.Count -gt 1 }) { + Write-Host "Duplicate step names in job '$job'" + return @() + } + return $steps + } + + [hashtable[]] GetCustomStepsFromAnchor([string] $job, [string] $anchorStep, [bool] $before) { + $steps = $this.GetStepsFromJob($job) + $anchorIdx = $steps.IndexOf($anchorStep) + if ($anchorIdx -lt 0) { + Write-Host "Cannot find anchor step '$anchorStep' in job '$job'" + return @() + } + $idx = $anchorIdx + $customSteps = @() + if ($before) { + while ($idx -gt 0 -and $steps[$idx-1] -like 'CustomStep*') { + $idx-- + } + if ($idx -ne $anchorIdx) { + $customSteps = @($steps[$idx..($anchorIdx-1)]) + # Reverse the order of the custom steps in order to apply in correct order from the anchor step + [array]::Reverse($customSteps) + } + } + else { + while ($idx -lt $steps.Count-1 -and $steps[$idx+1] -like 'CustomStep*') { + $idx++ + } + if ($idx -ne $anchorIdx) { + $customSteps = @($steps[($anchorIdx+1)..$idx]) + } + } + $result = @() + foreach($customStep in $customSteps) { + $stepContent = $this.Get("Jobs:/$($job):/steps:/- name: $($customStep)").content + $result += @(@{"Name" = $customStep; "Content" = $stepContent; "AnchorStep" = $anchorStep; "Before" = $before }) + } + return $result + } + + [hashtable[]] GetCustomStepsFromYaml([string] $job, [hashtable[]] $anchors) { + $steps = $this.GetStepsFromJob($job) + $result = @() + foreach($anchor in $anchors) { + $result += $this.GetCustomStepsFromAnchor($job, $anchor.Step, $anchor.Before) + } + foreach($step in $steps) { + if ($step -like 'CustomStep*') { + if (-not ($result | Where-Object { $_.Name -eq $step })) { + Write-Host "Custom step '$step' does not belong to a supported anchor" + } + } + } + return $result + } + + [void] AddCustomStepsToAnchor([string] $job, [hashtable[]] $customSteps, [string] $anchorStep, [bool] $before) { + $steps = $this.GetStepsFromJob($job) + if (!$steps) { + Write-Host "::Warning::Cannot find job '$job'" + return + } + $anchorIdx = $steps.IndexOf($anchorStep) + if ($anchorIdx -lt 0) { + Write-Host "::Warning::Cannot find anchor step '$anchorStep' in job '$job'" + return + } + foreach($customStep in $customSteps | Where-Object { $_.AnchorStep -eq $anchorStep -and $_.Before -eq $before }) { + if ($steps -contains $customStep.Name) { + Write-Host "Custom step '$($customStep.Name)' already exists in job '$job'" + } + else { + $anchorStart = 0 + $anchorCount = 0 + if ($this.Find("Jobs:/$($job):/steps:/- name: $($anchorStep)", [ref] $anchorStart, [ref] $anchorCount)) { + if ($before) { + $this.Insert($anchorStart-1, @('') + @($customStep.Content | ForEach-Object { " $_" })) + } + else { + $this.Insert($anchorStart+$anchorCount, @('') + @($customStep.Content | ForEach-Object { " $_" })) + } + } + } + # Use added step as anchor for next step + $anchorStep = $customStep.Name + } + } + + [void] AddCustomStepsToYaml([string] $job, [hashtable[]] $customSteps, [hashtable[]] $anchors) { + foreach($anchor in $anchors) { + $this.AddCustomStepsToAnchor($job, $customSteps, $anchor.Step, $anchor.Before) + } + } + + static [PSCustomObject] GetPermissionsFromArray([string[]] $permissionsArray) { + $permissions = [PSCustomObject]@{} + $permissionsArray | ForEach-Object { + $permissions | Add-Member -MemberType NoteProperty -Name $_.Split(':')[0].Trim() -Value $_.Split(':')[1].Trim() + } + return $permissions + } + + static [string[]] GetPermissionsArray([PSCustomObject] $permissions) { + $permissionsArray = @() + $permissions.PSObject.Properties.Name | ForEach-Object { + $permissionsArray += "$($_): $($permissions."$_")" + } + return $permissionsArray + } + + static [PSCustomObject] MergePermissions([PSCustomObject] $permissions, [PSCustomObject] $permissions2) { + $permissions2.PSObject.Properties.Name | ForEach-Object { + if ($permissions.PSObject.Properties.Name -eq $_) { + $permission = $permissions."$_" + $permission2 = $permissions2."$_" + if ($permission -eq 'write' -or $permission2 -eq 'write') { + $permissions."$_" = 'write' + } + elseif ($permission -eq 'read' -or $permission2 -eq 'read') { + $permissions."$_" = 'read' + } + else { + $permissions."$_" = 'none' + } + } + else { + $permissions | Add-Member -MemberType NoteProperty -Name $_ -Value $permissions2."$_" + } + } + return $permissions + } + + static [void] ApplyCustomizations([ref] $srcContent, [string] $yamlFile, [hashtable] $anchors) { + $srcYaml = [Yaml]::new($srcContent.Value.Split("`n")) + try { + $yaml = [Yaml]::Load($yamlFile) + } + catch { + return + } + # Merge permissions + Write-host "Merge permissions" + $srcPermissionsObj = $srcYaml.Get('permissions:/') + $yamlPermissionsObj = $yaml.Get('permissions:/') + if ($srcPermissionsObj -and $yamlPermissionsObj) { + $srcPermissions = [Yaml]::GetPermissionsFromArray($srcPermissionsObj.content) + $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) + if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { + $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + } + } + + # Apply cystom steps + Write-Host "Apply custom steps" + $filename = [System.IO.Path]::GetFileName($yamlFile) + if ($anchors.ContainsKey($filename)) { + $fileAnchors = $anchors."$filename" + foreach($job in $fileAnchors.Keys) { + # Locate custom steps in destination YAML + $customSteps = $yaml.GetCustomStepsFromYaml($job, $fileAnchors."$job") + if ($customSteps) { + $srcYaml.AddCustomStepsToYaml($job, $customSteps, $fileAnchors."$job") + } + } + } + # Locate custom jobs in destination YAML + Write-Host "Apply custom jobs" + $customJobs = @($yaml.GetCustomJobsFromYaml('CustomJob*')) + if ($customJobs) { + # Add custom jobs to template YAML + $srcYaml.AddCustomJobsToYaml($customJobs) + } + $srcContent.Value = $srcYaml.content -join "`n" + } } diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index c4439d7f4..399b8073d 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -709,17 +709,12 @@ function Set-ContentLF { [parameter(mandatory = $true, ValueFromPipeline = $false)] [string] $path, [parameter(mandatory = $true, ValueFromPipeline = $true)] - $content + [string] $content ) Process { $path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($path) - if ($content -is [array]) { - $content = $content -join "`n" - } - else { - $content = "$content".Replace("`r", "") - } + $content = "$content".Replace("`r", "").TrimEnd("`n") [System.IO.File]::WriteAllText($path, "$content`n") } } diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index ba86fd9e7..884228e99 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -81,7 +81,7 @@ try { $appBuild = $settings.appBuild $appRevision = $settings.appRevision - 'licenseFileUrl','codeSignCertificateUrl','*codeSignCertificatePassword','keyVaultCertificateUrl','*keyVaultCertificatePassword','keyVaultClientId','gitHubPackagesContext','applicationInsightsConnectionString' | ForEach-Object { + 'licenseFileUrl','codeSignCertificateUrl','*codeSignCertificatePassword','keyVaultCertificateUrl','*keyVaultCertificatePassword','keyVaultClientId','gitHubPackagesContext','applicationInsightsConnectionString','cicdAuthContext' | ForEach-Object { # Secrets might not be read during Pull Request runs if ($secrets.Keys -contains $_) { $value = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$_")) @@ -219,6 +219,19 @@ try { } $authContext = $null $environmentName = "" + if ($cicdAuthContext) { + $authContext = $cicdAuthContext | ConvertFrom-Json | ConvertTo-HashTable + if ($authContext.ContainsKey('environmentName')) { + $environmentName = $authContext.environmentName + $authContext.Remove('environmentName') + if ($environmentName -notlike 'https://*') { + $authContext = New-BcAuthContext @authContext + } + } + else { + Write-Host "::WARNING::CI/CD AuthContext is missing environmentName, ignoring cicdAuthContext secret." + } + } $CreateRuntimePackages = $false if ($settings.versioningStrategy -eq -1) { diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 62eba2435..c6a9e05d6 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -116,7 +116,7 @@ try { "appSourceAppRepo" = "$($config.githubOwner)/$($config.appSourceAppRepo)" } - if ($config.branch -eq 'preview') { + if ($config.branch -eq 'preview' -or $config.githubOwner -ne 'microsoft') { # When deploying to preview, we are NOT going to deploy to a branch in the AL-Go-Actions repository # Instead, we are going to have AL-Go-PTE and AL-Go-AppSource point directly to the SHA in AL-Go $dstOwnerAndRepo += @{ diff --git a/README.md b/README.md index 2ff4b733e..d67e0bf67 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Try out the [AL-Go workshop](https://aka.ms/algoworkshop) for an in-depth worksh 1. [Connect your GitHub repository to Power Platform](Scenarios/SetupPowerPlatform.md) 1. [How to set up Service Principal for Power Platform](Scenarios/SetupServicePrincipalForPowerPlatform.md) 1. [Try one of the Business Central and Power Platform samples](Scenarios/TryPowerPlatformSamples.md) +1. [Customizing AL-Go for GitHub](Scenarios/CustomizingALGoForGitHub.md) ## Migration scenarios diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 62f016758..56da4fd55 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -2,6 +2,33 @@ - It is now possible to skip the modification of dependency version numbers when running the Increment Version number workflow or the Create Release workflow +### New Settings + +- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: + + - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) + - **Source** (mandatory) - URL to a either a single file or a .zip file containing + - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. + - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) + +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Add custom steps to the AL-Go build workflow + +It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. + +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Indirect AL-Go template repository + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + ### New Versioning Strategy Setting versioning strategy to 3 will allow 3 segments of the version number to be defined in app.json and repoVersion. Only the 4th segment (Revision) will be defined by the GitHub [run_number](https://go.microsoft.com/fwlink/?linkid=2217416&clcid=0x409) for the CI/CD workflow. Increment version number and Create Release now also supports the ability to set a third segment to the RepoVersion and appversion in app.json. diff --git a/Scenarios/Contribute.md b/Scenarios/Contribute.md index cd385fa8b..0bca431d8 100644 --- a/Scenarios/Contribute.md +++ b/Scenarios/Contribute.md @@ -5,16 +5,16 @@ This section describes how to contribute to AL-Go. How to set up your own enviro You can do this in two ways: - Use a fork of AL-Go for GitHub in your own **personal GitHub account** in development mode -- Use 3 public repositories in your own **personal GitHub account** (AL-Go-PTE, AL-Go-AppSource and AL-Go-Actions, much like in production) +- Use 2 public repositories in your own **personal GitHub account** (AL-Go-PTE, AL-Go-AppSource and AL-Go-Actions, much like in production) ## Use a fork of AL-Go for GitHub in "development mode" 1. Fork the [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go) repository to your **personal GitHub account**. 1. You can optionally also create a branch in the AL-Go fork for the feature you are working on. -**https://github.com//AL-Go@** can now be used as a template in your AL-Go project when running _Update AL-Go System Files_ to use the actions/workflows from this fork. +**/AL-Go@** can now be used as a template in your AL-Go project when running _Update AL-Go System Files_ to use the actions/workflows from this fork. -## Use 3 public repositories in "production mode" +## Use 2 public repositories in "production mode" 1. Fork the [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go) repository to your **personal GitHub account**. 1. Navigate to [https://github.com/settings/tokens/new](https://github.com/settings/tokens/new) and create a new personal access token with **Full control of private repositories** and **workflow** permissions. @@ -22,15 +22,11 @@ You can do this in two ways: 1. In your personal fork of AL-Go, navigate to **Actions**, select the **Deploy** workflow and choose **Run Workflow**. 1. Using the default settings press **Run workflow**. Select the AL-Go branch to run from and the branch to deploy to. -Now you should have 3 new public repositories: +Now you should have 2 new public repositories: -- [https://github.com/yourGitHubUserName/AL-Go-Actions](https://github.com/yourGitHubUserName/AL-Go-Actions) - [https://github.com/yourGitHubUserName/AL-Go-AppSource](https://github.com/yourGitHubUserName/AL-Go-AppSource) - [https://github.com/yourGitHubUserName/AL-Go-PTE](https://github.com/yourGitHubUserName/AL-Go-PTE) -> \[!NOTE\] -> Deploying to a branch called **preview** will only update the two template repositories (and use your AL-Go project with the current SHA as actions repository). - You can optionally also create a branch in the AL-Go fork for the feature you are working on and then select that branch when running Deploy (both as **Use workflow from** and as **Branch to deploy to**). **yourGitHubUserName/AL-Go-PTE@yourBranch** or **yourGitHubUserName/AL-Go-AppSource@yourBranch** can now be used in your AL project when running Update AL-Go System Files to use the actions/workflows from this area for your AL project. @@ -38,7 +34,7 @@ You can optionally also create a branch in the AL-Go fork for the feature you ar Please ensure that all unit tests run and create a Pull Request against [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go). You are very welcome to run the end to end tests as well, but we will also run the end to end tests as part of the code review process. > \[!NOTE\] -> You can also deploy to a different branch in the 3 public repositories by specifying a branch name under **Branch to deploy** to when running the **Deploy** workflow. The branch you specify in **Use workflow from** indicates which branch in **your personal fork of the AL-Go repository** you publish to the 3 repositories. +> You can also deploy to a different branch in the 2 public repositories by specifying a branch name under **Branch to deploy** to when running the **Deploy** workflow. The branch you specify in **Use workflow from** indicates which branch in **your personal fork of the AL-Go repository** you publish to the 2 repositories. ## Pre-Commit @@ -57,9 +53,8 @@ In the e2eTests folder, in the AL-Go repository, there are 3 types of end to end - The scenarios folder contains a set of AL-Go scenarios, which tests specific functionality end to end. Every folder under e2eTests/scenarios, which contains a runtests.ps1 will be run as a scenario test, like: - UseProjectDependencies - create a repo with multiple projects and set **UseProjectDependencies** to modify CI/CD and other build workflows to build projects in the right order - GitHubPackages - create 3 repositories using GitHub Packages as dependency resolver and check that artifacts are built properly - - BuildModes - create a repository, set buildModes and test that generated artifacts are as expected. - - ReleaseBranches - testing that create release works, release branches are create and subsequently found correctly as previous build - SpecialCharacters - testing that various settings (+ publisher name and app name) can contain special national characters + - and more... In your personal fork, you can now run the end to end tests, if the following pre-requisites are available: @@ -80,7 +75,6 @@ You can also run the end to end tests directly from VS Code, by providing the fo |$global:E2EgitHubOwner| String | The GitHub owner of the test repositories (like `freddydk` or `microsoft`) | |$global:SecureE2EPAT| SecureString | A personal access token with workflow permissions | |$global:SecureAdminCenterApiToken| SecureString | Admin Center API Credentials | -|$global:SecureLicenseFileUrl| SecureString | Direct download URL to a license file | |$global:pteTemplate| String | URL for your PTE template (like `freddyk/AL-Go-PTE@main` or `freddydk/AL-Go@main\|Templates/Per Tenant Extension` for using your AL-Go fork directly) | |$global:appSourceTemplate| String | URL for your PTE template (like `freddyk/AL-Go-AppSource@main` or `freddydk/AL-Go@main\|Templates/AppSource App` for using your AL-Go fork directly) | diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md new file mode 100644 index 000000000..5dfad600c --- /dev/null +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -0,0 +1,301 @@ +# Customizing AL-Go for GitHub + +AL-Go for GitHub is a plug-and-play DevOps solution, intended to support 100% of the functionality needed by 90% of the people developing applications for Microsoft Dynamics 365 Business Central out-of-the-box. + +If AL-Go functionality out-of-the-box doesn't match your needs, you really have three options: + +1. Customize AL-Go for GitHub to fit your needs +1. Select another managed DevOps solution +1. Create your own DevOps solution from scratch (not recommended) + +Creating your own DevOps solution from scratch requires dedicated resources to develop and maintain workflows, processes etc. **This is not a small task**. There are many moving parts in a DevOps solution, which might require you to make changes to workflows and scripts over time and stay secure and having to maintain many repositories is tedious and time consuming, even when using templates and other advanced features. + +Microsoft will continuously develop and maintain AL-Go for GitHub and ensure that we always use the latest versions of GitHub actions, which are under our control. Microsoft will never add dependencies to any third party GitHub action, which are not under our control. + +Keeping your repositories up-to-date can be done manually or on a schedule (like Windows update really). You will be notified when an update is available and we recommend that you keep your repositories up-to-date at all time. If you make modifications to the AL-Go System Files (scripts and workflows) in your repository, in other ways than described in this document, these changes will be removed with the next AL-Go update. + +> \[!TIP\] +> If for some reason the updated version of AL-Go for GitHub doesn't work for you, we recommend that you file an issue [here](https://github.com/microsoft/AL-Go/issues) with a detailed description of the problem and full logs of the failing workflows. You can then revert back to the prior version of AL-Go for GitHub until the issue is resolved. +> +> It is important to get back to the mainstream version of AL-Go for GitHub as soon as the issue is resolved. + +There are three ways you can customize AL-Go for GitHub to fit your needs. You can + +1. customize the repository with custom scripts, workflows, jobs or steps following the guidelines below +1. create a customized repository and use this as your template repository (indirect template) +1. fork the AL-Go for GitHub and create your "own" version (not recommended) + +> \[!CAUTION\] +> The more you customize AL-Go for GitHub, the more likely you are to be broken by future updates to AL-Go for GitHub, meaning that you will have to update your customizations to match the changes in AL-Go for GitHub. + +## Customizing your repository + +There are several ways you can customize your AL-Go repository and ensure that the changes you make, will survive an update of AL-Go for GitHub. + +### Hide/Remove unused workflows + +By adding a setting called [`unusedALGoSystemFiles`](https://aka.ms/algosettings#unusedalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these system files are not used. Example: + +``` + "unusedALGoSystemFiles": [ + "AddExistingAppOrTestApp.yaml", + "CreateApp.yaml", + "CreatePerformanceTestApp.yaml", + "CreateTestApp.yaml", + "cloudDevEnv.ps1" + ] +``` + +This setting will cause AL-Go for GitHub to remove these files during the next update. Note that if you remove files like `_BuildALGoProject.yaml`, AL-Go will obviously stop working as intended - so please use with care. + +### Custom delivery + +You can setup [custom delivery](https://aka.ms/algosettings#customdelivery) in order to deliver your apps to locations not supported by AL-Go for GitHub out-of-the-box, by adding a custom delivery powershell script (named `.github/DeliverTo.ps1`) and a context secret (called `Context`) formatted as compressed json, you can define the delivery functionality as you like. Example: + +```powershell +Param([Hashtable] $parameters) + +Get-ChildItem -Path $parameters.appsFolder | Out-Host +$context = $parameters.context | ConvertFrom-Json +Write-Host "Token Length: $($context.Token.Length)" +``` + +In this example the context secret is assumed to contain a Token property. Read [this](https://aka.ms/algosettings#customdelivery) for more information. + +### Custom deployment + +You can setup [custom deployment](https://aka.ms/algosettings#customdeployment) to environment types not supported by AL-Go for GitHub out-of-the-box. You can also override deployment functionality to environment Type `SaaS` if you like. You can add an environment called `` and a `DeployTo` setting, defining which environment Type should be used. Example: + +```json + "Environments": [ + "" + ], + "DeployTo": { + "EnvironmentType": "" + } +``` + +You also need to create an AuthContext secret (called `_AuthContext`) and a powershell script (named `.github/DeployTo.ps1`), which defines the deployment functionality. Example: + +```powershell +Param([Hashtable] $parameters) + +$parameters | ConvertTo-Json -Depth 99 | Out-Host +$tempPath = Join-Path ([System.IO.Path]::GetTempPath()) ([GUID]::NewGuid().ToString()) +New-Item -ItemType Directory -Path $tempPath | Out-Null +Copy-AppFilesToFolder -appFiles $parameters.apps -folder $tempPath | Out-Null +Get-ChildItem -Path $tempPath -Filter *.app | Out-Host +$authContext = $parameters.authContext | ConvertFrom-Json +Write-Host "Token Length: $($authContext.Token.Length)" +``` + +In this example the AuthContext secret is assumed to contain a Token property. Read [this](https://aka.ms/algosettings#customdeployment) for more information. + +### Adding custom workflows + +If you add new workflows to the `.github/workflows` folder, which is unknown to AL-Go for GitHub, AL-Go will leave them un-touched. These workflows needs to follow standard GitHub Actions schema (yaml) and can be triggered as any other workflows. Example: + +```yaml +name: 'Create Build Tag' + +on: + workflow_run: + workflows: [' CI/CD','CI/CD'] + types: [completed] + branches: [ 'main' ] + +run-name: "[${{ github.ref_name }}] Create build tag" + +permissions: read-all + +jobs: + CreateTag: + if: github.event.workflow_run.conclusion == 'success' + runs-on: windows-latest + steps: + - name: mystep + run: | + Write-Host "Create tag" +``` + +It is recommended to prefix your workflows with `my`, `our`, your name or your organization name in order to avoid that the workflow suddenly gets overridden by a new workflow in AL-Go for GitHub. The above workflow is a real example from [here](https://github.com/microsoft/BCApps/blob/main/.github/workflows/CreateBuildTag.yaml). + +> \[!CAUTION\] +> This workflow gets triggered when the CI/CD workflow has completed. Note that the name of the CI/CD workflow currently is prefixed with a space, this space will very likely be removed in the future, which is why we specify both names in this example. Obviously this workflow would break if we decide to rename the CI/CD workflow to something different. + +### Adding custom scripts + +You can add custom powershell scripts under the .github folder for repository scoped scripts or in the .AL-Go folder for project scoped scripts. Specially named scripts in the .AL-Go folder can override standard functionality in AL-Go for GitHub workflows. A list of these script overrides can be found [here](https://aka.ms/algosettings#customdeployment). Scripts under the .github folder can be used in custom workflows instead of using inline scripts inside the workflow. + +One example of a script override is the NewBcContainer override used in the System Application project in BCApps (can be found [here](https://github.com/microsoft/BCApps/blob/main/build/projects/System%20Application/.AL-Go/NewBcContainer.ps1)). This override looks like: + +```powershell +Param([Hashtable] $parameters) + +$script = Join-Path $PSScriptRoot "../../../scripts/NewBcContainer.ps1" -Resolve +. $script -parameters $parameters +``` + +Which basically launches a script located in the script folder in the repository for creating the build container needed for building and testing the System Application. + +> \[!CAUTION\] +> Script overrides will almost certainly be broken in the future. The current script overrides is very much tied to the current implementation of the `Run-AlPipeline` function in BcContainerHelper. In the future, we will move this functionality to GitHub actions and no longer depend on BcContainerHelper and Run-AlPipeline. At that time, these script overrides will have to be changed to follow the new implementation. + + + +### Adding custom workflows and/or scripts using a URL + +By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings#customalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these files should be included in the update. Example: + +``` + "customALGoSystemFiles": [ + { + "Destination": ".AL-Go/", + "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1" + }, + { + "Destination": ".AL-Go/NewBcContainer.ps1", + "Source": "https://raw.githubusercontent.com/microsoft/BCApps/main/build/scripts/NewBcContainer.ps1" + }, + { + "Destination": ".github/", + "Source": "https://github.com/freddydk/CustomALGoSystemFiles/archive/refs/heads/main.zip", + "FileSpec": "*/.github/*", + "Recurse": true + } + ] +``` + +`customALGoSystemFiles` is an array of objects, which currently can have 4 properties: + +| Property | Description | Mandatory | Default | +| :-- | :-- | :-: | :-- | +| Destination | Path in which the file should be placed. Can include the filename if the source doesn't point to a .zip file, must include a terminating / or \\ if a filename is not included. | Yes | | +| Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. | Yes | | +| FileSpec | If the source URL points to a .zip file, this property can specify which files to include. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | +| Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | + +This setting will cause AL-Go for GitHub to include these files during the next update. + +> \[!WARNING\] +> You can override existing AL-Go for GitHub system files this way, please prefix files in your repository with `my` or your organization name (except for DeployTo and DeliverTo) in order to avoid overriding future workflows from AL-Go for GitHub. + +> \[!NOTE\] +> If the destination is in the .AL-Go folder, the file(s) will be copied to all .AL-Go folders in multi-project repositories. + +### Adding custom jobs + +You can also add custom jobs to any of the existing AL-Go for GitHub workflows. Custom jobs can depend on other jobs and other jobs can made to depend on custom jobs. Custom jobs needs to be named `CustomJob`, but can specify another name to be shown in the UI. Example: + +```yaml + CustomJob-CreateBuildTag: + name: Create Build Tag + needs: [ Initialization, Build ] + if: (!cancelled()) && (needs.Build.result == 'success') + runs-on: [ ubuntu-latest ] + steps: + - name: Create Tag + run: | + Write-Host "Create Tag" + + PostProcess: + needs: [ Initialization, Build2, Build1, Build, Deploy, Deliver, DeployALDoc, CustomJob-CreateBuildTag ] + if: (!cancelled()) + runs-on: [ windows-latest ] + steps: + ... +``` + +Adding a custom job like this, will cause this job to run simultaneously with the deploy and the deliver jobs. + +> \[!NOTE\] +> All custom jobs will be moved to the tail of the yaml file when running Update AL-Go System Files, but dependencies to/from the custom jobs will be maintained. + +> \[!CAUTION\] +> Custom jobs might be broken if the customized AL-Go for GitHub workflow has been refactored and the referenced jobs have been renamed. + +### Adding custom steps + +You can also add custom steps to AL-Go for GitHub Workflows, but only in pre-defined anchor-points. The reason for only allowing custom steps at pre-defined anchor-points is that we want to limit the number of places where steps can be added in order to have some level of freedom to refactor, develop and maintain the AL-Go for GitHub workflows, without breaking customizations constantly. + +At this time, the anchor-points where you can add custom steps are: + +| Workflow | Job | Step | Before or AFter | +| :-- | :-- | :-- | :-: | +| \_BuildALGoProject.yaml | BuildALGoProject | Read settings | After | +| | | Read secrets | After | +| | | Build | Before | +| | | Read secrets | After | +| | | Cleanup | Before | + +The custom step needs to be named `CustomStep` and if inserted in any of the specified anchor-points, it will be maintained after running Update AL-Go System Files. An example of a custom step could be a step, which modifies settings based on some business logic + +```yaml + - name: CustomStep-ModifySettings + run: | + $settings = $env:Settings | ConvertFrom-Json + $settings.artifact = Invoke-RestMethod -Method GET -UseBasicParsing -Uri "https://bca-url-proxy.azurewebsites.net/bca-url/sandbox/us?select=weekly&doNotRedirect=true" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "Settings=$($$settings | ConvertTo-Json -Depth 99 -Compress)" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "artifact=$($settings.artifact)" +``` + +> \[!TIP\] +> Create a feature request [here](https://github.com/microsoft/AL-Go/issues/new?assignees=&labels=enhancement&projects=&template=enhancement.yaml&title=%5BEnhancement%5D%3A+) with a description on where you would like additional anchor-points and what you want to use it for. + +> \[!CAUTION\] +> Please be aware that changes to AL-Go for GitHub might break with future versions of AL-Go for GitHub. We will of course try to keep these breaking changes to a minimum, but the only way you can be sure to NOT be broken is by NOT customizing AL-Go for GitHub. + +### Modifying workflow permissions + +If any of your custom jobs require permissions, which exceeds the permissions already assigned in the workflow, then these permissions can be specified directly on the custom job. + +If any of your custom steps require permissions, which exceeds the permissions already assigned in the workflow, you can modify the permissions of the workflow and assign additional permissions. AL-Go for GitHub will not allow you to remove permissions, which might be needed in other steps/jobs, but additional permissions will be included when running Update AL-Go System Files. + +## Using indirect templates + +If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. + +> \[!NOTE\] +> Indirect templates can be public or private. If you are using a private indirect template, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. + +Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. **UnusedALGoSystemFiles** and **CustomALGoSystemFiles** will NOT be copied from the indirect template, they will be applied during Update AL-Go System Files. + +> \[!TIP\] +> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a **GhTokenWorkflow** secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. + +> \[!NOTE\] +> If you use the indirect template as a GitHub template for creating the repository, by clicking use this template in your indirect template - then you need to re-specify the indirect Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the indirect template as it's template repository. + +Repositories based on your indirect template will notify you that changes are available for your AL-Go System Files when you update the indirect template only. You will not be notified when new versions of AL-Go for GitHub is released in every repository - only in the indirect template repository. + +> \[!WARNING\] +> You should ensure that your indirect template repository is kept up-to-date with the latest changes in AL-Go for GitHub. + +> \[!TIP\] +> You can setup the Update AL-Go System Files workflow to run on a schedule to uptake new releases of AL-Go for GitHub regularly. + +## Forking AL-Go for GitHub and making your "own" **public** version + +Using a fork of AL-Go for GitHub to have your "own" public version of AL-Go for GitHub gives you the maximum customization capabilities. It does however also come with the most work. + +> \[!NOTE\] +> When customizing AL-Go for GitHub using a fork, your customizations are public and will be visible to everyone. For more information, [read this](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-permissions-and-visibility-of-forks). + +There are two ways of forking AL-Go for GitHub. You can fork the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository or you can fork the template repositories [AL-Go PTE](https://github.com/microsoft/AL-Go-PTE) and/or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource). + +For simple changes to the templates, you can fork the template repositories and make the changes directly in your fork. Note that we do not accept any pull requests to the template repositories as they are deployed from the main AL-Go repository. We do not actually develop anything in the template repositories ourself. In the template repositories you will find a branch for every version of AL-Go we have shipped. The main branch is the latest version and the preview branch is the next version. You can customize the preview branch and/or the main branch and then use your fork as the template repository when running Update AL-Go System Files from your app repositories. + +> \[!NOTE\] +> We do NOT accept pull requests to the template repositories. You need to follow the guidelines [here](Contribute.md) in order to contribute to AL-Go development. + +> \[!TIP\] +> When forking the template repositories, you should include all branches in order to be able to use either the latest version of AL-Go or the preview version of AL-Go. + +When forking the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository, you are basically developing AL-Go in the same way as we are doing in Microsoft. Please follow the guidelines [here](Contribute.md) on how to develop. This gives you maximum customization capabilities, but if your changes are not being contributed to AL-Go, then you will have to merge our changes all the time. + +> \[!CAUTION\] +> We strongly suggest that you keep your changes to a minimum and that you keep your fork up-to-date with the latest changes of AL-Go for GitHub at all time. + +______________________________________________________________________ + +[back](../README.md) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 0a590431e..78c101f1e 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -4,6 +4,8 @@ The behavior of AL-Go for GitHub is very much controlled by settings and secrets To learn more about the secrets used by AL-Go for GitHub, please navigate to [Secrets](secrets.md). + + ## Where are the settings located Settings can be defined in GitHub variables or in various settings file. An AL-Go repository can consist of a single project (with multiple apps) or multiple projects (each with multiple apps). Settings can be applied on the project level or on the repository level. Multiple projects in a single repository are comparable to multiple repositories; they are built, deployed, and tested separately. All apps in each project (single or multiple) are built together in the same pipeline, published and tested together. If a repository is multiple projects, each project is stored in a separate folder in the root of the repository. @@ -24,6 +26,8 @@ When running a workflow or a local script, the settings are applied by reading s 1. `.AL-Go/.settings.json` is the **user-specific settings file**. This option is rarely used, but if you have special settings, which should only be used for one specific user (potentially in the local scripts), these settings can be added to a settings file with the name of the user followed by `.settings.json`. + + ## Basic Project settings | Name | Description | Default value | @@ -81,6 +85,8 @@ The repository settings are only read from the repository settings file (.github | useGitSubmodules | If your repository is using Git Submodules, you can set the `useGitSubmodules` setting to `"true"` or `"recursive"` in order to use these submodules during build workflows. If `useGitSubmodules` is not set, git submodules are not initialized. If the submodules reside in private repositories, you need to define a `gitSubmodulesToken` secret. Read [this](https://aka.ms/algosecrets#gitSubmodulesToken) for more information. | | commitOptions | If you want more control over how AL-Go creates pull requests or commits changes to the repository you can define `commitOptions`. It is a structure defining how you want AL-Go to handle automated commits or pull requests coming from AL-Go (e.g. for Update AL-Go System Files). The structure contains the following properties:
`messageSuffix` : A string you want to append to the end of commits/pull requests created by AL-Go. This can be useful if you are using the Azure Boards integration (or similar integration) to link commits to work items.
`pullRequestAutoMerge` : A boolean defining whether you want AL-Go pull requests to be set to auto-complete. This will auto-complete the pull requests once all checks are green and all required reviewers have approved.
`pullRequestLabels` : A list of labels to add to the pull request. The labels need to be created in the repository before they can be applied.
If you want different behavior in different AL-Go workflows you can add the `commitOptions` setting to your [workflow-specific settings files](https://github.com/microsoft/AL-Go/blob/main/Scenarios/settings.md#where-are-the-settings-located). | + + ## Advanced settings | Name | Description | Default value | @@ -132,6 +138,8 @@ The repository settings are only read from the repository settings file (.github | appSourceContextSecretName | This setting specifies the name (**NOT the secret**) of a secret containing a json string with ClientID, TenantID and ClientSecret or RefreshToken. If this secret exists, AL-Go will can upload builds to AppSource validation. | AppSourceContext | | keyVaultCertificateUrlSecretName
keyVaultCertificatePasswordSecretName
keyVaultClientIdSecretName | If you want to enable KeyVault access for your AppSource App, you need to provide 3 secrets as GitHub Secrets or in the Azure KeyVault. The names of those secrets (**NOT the secrets**) should be specified in the settings file with these 3 settings. Default is to not have KeyVault access from your AppSource App. Read [this](EnableKeyVaultForAppSourceApp.md) for more information. | | + + ## Conditional Settings In any of the settings files, you can add conditional settings by using the ConditionalSettings setting. @@ -183,6 +191,14 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza > \[!NOTE\] > You can have conditional settings on any level and all conditional settings which has all conditions met will be applied in the order of settings file + appearance. + + +# Expert level + +The settings and functionality in the expert section might require knowledge about GitHub Workflows/Actions, YAML, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. + +Please read the release notes carefully when installing new versions of AL-Go for GitHub. + ## Expert settings (rarely used) | Name | Description | Default value | @@ -202,12 +218,13 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza | memoryLimit | Specifies the memory limit for the build container. By default, this is left to BcContainerHelper to handle and will currently be set to 8G | 8G | | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | \[ \] | +| customALGoSystemFiles | An array of objects, specifying custom AL-Go System Files. Each object must contain **Destination** and **Source** url and if the source url points to a .zip file, you can also add **FileSpec** and **Recurse**. See more [here](./CustomizingALGoForGitHub.md#customALGoSystemFiles) | \[ \] | -# Expert level + ## Custom Delivery -You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named DeliverTo\*.ps1 in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. +You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named `DeliverTo.ps1` in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. Beside the script, you also need to create a secret called `Context`, formatted as compressed json, containing delivery information for your delivery target. ### DeliverToSharePoint.ps1 @@ -219,6 +236,7 @@ Param( Write-Host "Current project path: $($parameters.project)" Write-Host "Current project name: $($parameters.projectName)" Write-Host "Delivery Type (CD or Release): $($parameters.type)" +Write-Host "Delivery Context: $($parameters.context)" Write-Host "Folder containing apps: $($parameters.appsFolder)" Write-Host "Folder containing test apps: $($parameters.testAppsFolder)" Write-Host "Folder containing dependencies (requires generateDependencyArtifact set to true): $($parameters.dependenciesFolder)" @@ -246,6 +264,8 @@ Here are the parameters to use in your custom script: | `$parameters.testAppsFolders` | The folders that contain the build artifacts from all builds (from different build modes) of the test apps in the AL-Go project | AllProjects_MyProject-main-TestApps-1.0.0.0, AllProjects_MyProject-main-CleanTestApps-1.0.0.0 | | `$parameters.dependenciesFolders` | The folders that contain the dependencies of the AL-Go project for all builds (from different build modes) | AllProjects_MyProject-main-Dependencies-1.0.0.0, AllProjects_MyProject-main-CleanDependencies-1.0.0.0 | + + ## Custom Deployment You can override existing AL-Go Deployment functionality or you can define your own custom deployment mechanism for AL-Go for GitHub. By specifying a PowerShell script named `DeployTo.ps1` in the .github folder. Default Environment Type is SaaS, but you can define your own type by specifying EnvironmentType in the `DeployTo` setting. The following example will create a script, which would be called by CI/CD and Publish To Environment, when EnvironmentType is set to OnPrem. @@ -296,14 +316,19 @@ Here are the parameters to use in your custom script: | `$parameters."runs-on"` | GitHub runner to be used to run the deployment script | windows-latest | | `$parameters."shell"` | Shell used to run the deployment script, pwsh or powershell | powershell | + + ## Run-AlPipeline script override AL-Go for GitHub utilizes the Run-AlPipeline function from BcContainerHelper to perform the actual build (compile, publish, test etc). The Run-AlPipeline function supports overriding functions for creating containers, compiling apps and a lot of other things. This functionality is also available in AL-Go for GitHub, by adding a file to the .AL-Go folder, you automatically override the function. +Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the future might break the usage of these overrides. + | Override | Description | | :-- | :-- | +| PipelineInitialize.ps1 | Initialize the pipeline | | DockerPull.ps1 | Pull the image specified by the parameter $imageName | | NewBcContainer.ps1 | Create the container using the parameters transferred in the $parameters hashtable | | ImportTestToolkitToBcContainer.ps1 | Import the test toolkit apps specified by the $parameters hashtable | @@ -321,6 +346,7 @@ This functionality is also available in AL-Go for GitHub, by adding a file to th | BackupBcContainerDatabases | Backup Databases in container for subsequent restore(s) | | RestoreDatabasesInBcContainer | Restore Databases in container | | InstallMissingDependencies | Install missing dependencies | +| PipelineFinalize.ps1 | Finalize the pipeline | ## BcContainerHelper settings @@ -328,6 +354,8 @@ The repo settings file (.github\\AL-Go-Settings.json) can contain BcContainerHel Settings, which might be relevant to set in the settings file includes +Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the future might break the usage of these overrides. + | Setting | Description | Default | | :-- | :-- | :-- | | baseUrl | The Base Url for the online Business Central Web Client. This should be changed when targetting embed apps. | [https://businesscentral.dynamics.com](https://businesscentral.dynamics.com) | @@ -338,12 +366,93 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | \[ \] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | + + +## Custom jobs in AL-Go for GitHub workflows + +Adding a custom job to any AL-Go for GitHub workflow is done by adding a job with the name `CustomJob` to the end of an AL-Go for GitHub workflow, like this: + +``` + CustomJob-PrepareDeploy: + name: My Job + needs: [ Build ] + runs-on: [ ubuntu-latest ] + defaults: + run: + shell: pwsh + steps: + - name: This is my job + run: | + Write-Host "This is my job" +``` + +In the `needs` property, you specify which jobs should be complete before this job is run. If you require this job to run before other AL-Go for GitHub jobs are complete, you can add the name of this job in the `needs` property of that job, like: + +``` + Deploy: + needs: [ Initialization, Build, CustomJob-PrepareDeploy ] + if: always() && needs.Build.result == 'Success' && needs.Initialization.outputs.environmentCount > 0 + strategy: ${{ fromJson(needs.Initialization.outputs.environmentsMatrixJson) }} +``` + +Custom jobs will be preserved when running Update AL-Go System Files. + +**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). + + + +## Custom steps in the \_BuildALGoProject workflow + +Adding a custom step is done by adding a step with the name `CustomStep` to the \_BuildALGoProject.yaml workflow at one of these anchor points: + +- Before Read Settings +- Before Read Secrets +- Before or After Build +- Before Cleanup + +Example, insert the following step before the Build step: + +``` + - name: CustomStep that will run before the Build step + run: | + Write-Host "before build" + + - name: Build + uses: ... +``` + +Custom steps will be preserved when running Update AL-Go System Files. + +**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). + + + +## Indirect template repositories + +If you are utilizing script overrides, custom jobs, custom steps, custom delivery or like in many repositories, you might want to take advantage of the indirect template repository feature. + +An indirect template repository is an AL-Go for GitHub repository (without any apps), which is used as a template for the remaining AL-Go for GitHub repositories. As an example, if you are using a custom delivery script, which you want to have in all your repositories, you can create an empty AL-Go for GitHub repository, place the delivery script in the .github folder and use that repository as a template when running Update AL-Go system files in your other repositories. + +This would make sure that all repositories would have this script (and updated versions of the script) in the future. + +The items, which are currently supported from indirect template repositories are: + +- Repository script overrides in the .github folder +- Project script overrides in the .AL-Go folder +- Custom workflows in the .github/workflows folder +- Custom jobs in any AL-Go for GitHub workflow +- Custom steps in the \_BuildALGoProject workflow +- New repository settings +- New project settings + +**Note** that an AL-Go for GitHub indirect template repository can be private or public. + ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. > \[!NOTE\] -> Our goal is to never break repositories, which are using AL-Go for GitHub as their template. We almost certainly will break you if you create local modifications to scripts and pipelines. +> Our goal is to never break repositories, which are using standard AL-Go for GitHub as their template. We almost certainly will break you at some point in time if you create local modifications to scripts and pipelines. ______________________________________________________________________ diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index 7c3bbb48a..dd47ec02d 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -151,10 +151,19 @@ jobs: shell: powershell get: templateUrl + - name: Read secrets + id: ReadSecrets + uses: microsoft/AL-Go-Actions/ReadSecrets@main + with: + shell: powershell + gitHubSecrets: ${{ toJson(secrets) }} + getSecrets: 'ghTokenWorkflow' + - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true @@ -180,10 +189,11 @@ jobs: publishArtifacts: ${{ github.ref_name == 'main' || startswith(github.ref_name, 'release/') || startswith(github.ref_name, 'releases/') || needs.Initialization.outputs.deliveryTargetsJson != '[]' || needs.Initialization.outputs.environmentCount > 0 }} signArtifacts: true useArtifactCache: true + needsContext: ${{ toJson(needs) }} DeployALDoc: needs: [ Initialization, Build ] - if: (!cancelled()) && needs.Build.result == 'Success' && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' + if: (!cancelled()) && (needs.Build.result == 'success' || needs.Build.result == 'skipped') && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' runs-on: [ windows-latest ] name: Deploy Reference Documentation permissions: diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index b9d189964..ee3bc8ca0 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -103,7 +103,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'TokenForPush' + getSecrets: 'TokenForPush,GhTokenWorkflow' useGhTokenWorkflowForPush: '${{ github.event.inputs.useGhTokenWorkflow }}' - name: Determine Projects @@ -116,6 +116,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/Current.yaml b/Templates/AppSource App/.github/workflows/Current.yaml index bd0e7df50..c46278f18 100644 --- a/Templates/AppSource App/.github/workflows/Current.yaml +++ b/Templates/AppSource App/.github/workflows/Current.yaml @@ -100,6 +100,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'Current' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/NextMajor.yaml b/Templates/AppSource App/.github/workflows/NextMajor.yaml index c244640ed..9f3a73905 100644 --- a/Templates/AppSource App/.github/workflows/NextMajor.yaml +++ b/Templates/AppSource App/.github/workflows/NextMajor.yaml @@ -100,6 +100,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMajor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/NextMinor.yaml b/Templates/AppSource App/.github/workflows/NextMinor.yaml index ea9bf6736..693e202bf 100644 --- a/Templates/AppSource App/.github/workflows/NextMinor.yaml +++ b/Templates/AppSource App/.github/workflows/NextMinor.yaml @@ -100,6 +100,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMinor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml b/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml index 2d452db89..15482ee49 100644 --- a/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml +++ b/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml @@ -99,6 +99,7 @@ jobs: secrets: 'licenseFileUrl,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'PR${{ github.event.number }}' + needsContext: ${{ toJson(needs) }} StatusCheck: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 430b20e01..c8daa469f 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -65,10 +65,10 @@ jobs: - name: Override templateUrl env: - templateUrl: ${{ github.event.inputs.templateUrl }} + newTemplateUrl: ${{ github.event.inputs.templateUrl }} run: | $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 - $templateUrl = $ENV:templateUrl + $templateUrl = $ENV:newTemplateUrl if ($templateUrl) { Write-Host "Using Template Url: $templateUrl" Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "templateUrl=$templateUrl" diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index a000dc58c..6bd3172e9 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -67,6 +67,10 @@ on: description: Flag determining whether to use the Artifacts Cache type: boolean default: false + needsContext: + description: JSON formatted needs context + type: string + default: '{}' permissions: actions: read @@ -106,7 +110,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,-gitSubmodulesToken' + getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,-gitSubmodulesToken,cicdAuthContext' - name: Checkout Submodules if: env.useGitSubmodules != 'false' && env.useGitSubmodules != '' @@ -148,6 +152,7 @@ jobs: env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} + NeedsContext: ${{ inputs.needsContext }} with: shell: ${{ inputs.shell }} artifact: ${{ env.artifact }} diff --git a/Templates/AppSource App/README.md b/Templates/AppSource App/README.md index ab1063e63..ea984de77 100644 --- a/Templates/AppSource App/README.md +++ b/Templates/AppSource App/README.md @@ -2,8 +2,6 @@ This template repository can be used for managing AppSource Apps for Business Central. -[![Use this template](https://github.com/microsoft/AL-Go/assets/10775043/ca1ecc85-2fd3-4ab5-a866-bd2e7e80259d)](https://github.com/new?template_name=AL-Go-AppSource&template_owner=microsoft) - Please go to https://aka.ms/AL-Go to learn more. ## Contributing diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index c95d9be45..f4ff2a952 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -151,10 +151,19 @@ jobs: shell: powershell get: templateUrl + - name: Read secrets + id: ReadSecrets + uses: microsoft/AL-Go-Actions/ReadSecrets@main + with: + shell: powershell + gitHubSecrets: ${{ toJson(secrets) }} + getSecrets: 'ghTokenWorkflow' + - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true @@ -180,6 +189,7 @@ jobs: publishArtifacts: ${{ github.ref_name == 'main' || startswith(github.ref_name, 'release/') || startswith(github.ref_name, 'releases/') || needs.Initialization.outputs.deliveryTargetsJson != '[]' || needs.Initialization.outputs.environmentCount > 0 }} signArtifacts: true useArtifactCache: true + needsContext: ${{ toJson(needs) }} BuildPP: needs: [ Initialization ] @@ -197,7 +207,7 @@ jobs: DeployALDoc: needs: [ Initialization, Build ] - if: (!cancelled()) && needs.Build.result == 'Success' && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' + if: (!cancelled()) && (needs.Build.result == 'success' || needs.Build.result == 'skipped') && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' runs-on: [ windows-latest ] name: Deploy Reference Documentation permissions: diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index b9d189964..ee3bc8ca0 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -103,7 +103,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'TokenForPush' + getSecrets: 'TokenForPush,GhTokenWorkflow' useGhTokenWorkflowForPush: '${{ github.event.inputs.useGhTokenWorkflow }}' - name: Determine Projects @@ -116,6 +116,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/Current.yaml b/Templates/Per Tenant Extension/.github/workflows/Current.yaml index bd0e7df50..c46278f18 100644 --- a/Templates/Per Tenant Extension/.github/workflows/Current.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/Current.yaml @@ -100,6 +100,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'Current' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml b/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml index c244640ed..9f3a73905 100644 --- a/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml @@ -100,6 +100,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMajor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml b/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml index ea9bf6736..693e202bf 100644 --- a/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml @@ -100,6 +100,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMinor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml b/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml index 2d452db89..15482ee49 100644 --- a/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml @@ -99,6 +99,7 @@ jobs: secrets: 'licenseFileUrl,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'PR${{ github.event.number }}' + needsContext: ${{ toJson(needs) }} StatusCheck: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 430b20e01..c8daa469f 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -65,10 +65,10 @@ jobs: - name: Override templateUrl env: - templateUrl: ${{ github.event.inputs.templateUrl }} + newTemplateUrl: ${{ github.event.inputs.templateUrl }} run: | $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 - $templateUrl = $ENV:templateUrl + $templateUrl = $ENV:newTemplateUrl if ($templateUrl) { Write-Host "Using Template Url: $templateUrl" Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "templateUrl=$templateUrl" diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index a000dc58c..6bd3172e9 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -67,6 +67,10 @@ on: description: Flag determining whether to use the Artifacts Cache type: boolean default: false + needsContext: + description: JSON formatted needs context + type: string + default: '{}' permissions: actions: read @@ -106,7 +110,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,-gitSubmodulesToken' + getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,-gitSubmodulesToken,cicdAuthContext' - name: Checkout Submodules if: env.useGitSubmodules != 'false' && env.useGitSubmodules != '' @@ -148,6 +152,7 @@ jobs: env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} + NeedsContext: ${{ inputs.needsContext }} with: shell: ${{ inputs.shell }} artifact: ${{ env.artifact }} diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml index b323083aa..d344768c4 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml @@ -41,6 +41,10 @@ on: required: false type: string +permissions: + contents: read + actions: read + env: ALGoOrgSettings: ${{ vars.ALGoOrgSettings }} ALGoRepoSettings: ${{ vars.ALGoRepoSettings }} diff --git a/Templates/Per Tenant Extension/README.md b/Templates/Per Tenant Extension/README.md index 81dd89505..45baaa734 100644 --- a/Templates/Per Tenant Extension/README.md +++ b/Templates/Per Tenant Extension/README.md @@ -2,8 +2,6 @@ This template repository can be used for managing AppSource Apps for Business Central. -[![Use this template](https://github.com/microsoft/AL-Go/assets/10775043/ca1ecc85-2fd3-4ab5-a866-bd2e7e80259d)](https://github.com/new?template_name=AL-Go-PTE&template_owner=microsoft) - Please go to https://aka.ms/AL-Go to learn more. ## Contributing diff --git a/Tests/CheckForUpdates.Action.Test.ps1 b/Tests/CheckForUpdates.Action.Test.ps1 index 0fd0517a5..393b1a7aa 100644 --- a/Tests/CheckForUpdates.Action.Test.ps1 +++ b/Tests/CheckForUpdates.Action.Test.ps1 @@ -121,5 +121,34 @@ Describe "CheckForUpdates Action Tests" { $permissionsContent.content[1].Trim() | Should -be 'actions: read' } + It 'Test YamlClass Customizations' { + . (Join-Path $scriptRoot "yamlclass.ps1") + + $customizedYaml = [Yaml]::load((Join-Path $PSScriptRoot 'CustomizedYamlSnippet.txt')) + $yaml = [Yaml]::load((Join-Path $PSScriptRoot 'YamlSnippet.txt')) + + # Get Custom jobs from yaml + $customJobs = $customizedYaml.GetCustomJobsFromYaml('CustomJob*') + $customJobs | Should -Not -BeNullOrEmpty + $customJobs.Count | Should -be 1 + + # Get Custom steps from yaml + $customStep1 = $customizedYaml.GetCustomStepsFromAnchor('Initialization', 'Read settings', $true) + $customStep1 | Should -Not -BeNullOrEmpty + $customStep1.Count | Should -be 2 + + $customStep2 = $customizedYaml.GetCustomStepsFromAnchor('Initialization', 'Read settings', $false) + $customStep2 | Should -Not -BeNullOrEmpty + $customStep2.Count | Should -be 1 + + # Apply Custom jobs and steps to yaml + $yaml.AddCustomJobsToYaml($customJobs) + $yaml.AddCustomStepsToAnchor('Initialization', $customStep1, 'Read settings', $true) + $yaml.AddCustomStepsToAnchor('Initialization', $customStep2, 'Read settings', $false) + + # Check if new yaml content is equal to customized yaml content + ($yaml.content -join "`r`n") | Should -be ($customizedYaml.content -join "`r`n") + } + # Call action } diff --git a/Tests/CustomizedYamlSnippet.txt b/Tests/CustomizedYamlSnippet.txt new file mode 100644 index 000000000..0ccb32cb5 --- /dev/null +++ b/Tests/CustomizedYamlSnippet.txt @@ -0,0 +1,93 @@ +name: 'CI/CD' + +on: + workflow_dispatch: + workflow_run: + workflows: ["Pull Request Handler"] + types: + - completed + push: + paths-ignore: + - '**.md' + - '.github/workflows/*.yaml' + - '!.github/workflows/CICD.yaml' + branches: [ 'main', 'release/*', 'feature/*' ] + +run-name: ${{ fromJson(format('["","Check pull request from {1}/{2}{0} {3}"]',':',github.event.workflow_run.head_repository.owner.login,github.event.workflow_run.head_branch,github.event.workflow_run.display_title))[github.event_name == 'workflow_run'] }} + +permissions: + contents: read + actions: read + pull-requests: write + checks: write + +defaults: + run: + shell: powershell + +env: + workflowDepth: 1 + +jobs: + Initialization: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + runs-on: [ windows-latest ] + outputs: + telemetryScopeJson: ${{ steps.init.outputs.telemetryScopeJson }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: true + + - name: Initialize the workflow + id: init + uses: microsoft/AL-Go-Actions/WorkflowInitialize@main + with: + shell: powershell + + - name: CustomStep-MyStep1 + run: | + Write-Host 'My own step1!' + + - name: CustomStep-MyStepX + run: | + Write-Host 'Some stepX!' + + - name: Read settings + id: ReadSettings + uses: microsoft/AL-Go-Actions/ReadSettings@main + with: + shell: powershell + + - name: CustomStep-MyStep2 + run: | + Write-Host 'My own step2!' + + CheckForUpdates: + runs-on: [ windows-latest ] + needs: [ Initialization, CustomJob-MyJob ] + if: github.event_name != 'workflow_run' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read settings + uses: microsoft/AL-Go-Actions/ReadSettings@main + with: + shell: powershell + get: templateUrl + + - name: Check for updates to AL-Go system files + uses: microsoft/AL-Go-Actions/CheckForUpdates@main + with: + shell: powershell + templateUrl: ${{ env.templateUrl }} + + CustomJob-MyJob: + needs: [ Initialization ] + runs-on: [ windows-latest ] + steps: + - name: MyStep + run: | + Write-Host 'My own job!' diff --git a/Workshop/Index.md b/Workshop/Index.md index a87856e6e..3c702fc63 100644 --- a/Workshop/Index.md +++ b/Workshop/Index.md @@ -22,6 +22,14 @@ This workshop shows you how to take advantage of the functionality, which is pro 1. [The Development Process](TheDevelopmentProcess.md) - *FUTURE TOPIC: The recommended way to work with feature branches, pull requests, code reviews and branch protection rules.* 1. [Keeping your Repository Up-to-date](KeepUpToDate.md) - *FUTURE TOPIC: Updating AL-Go for GitHub to the latest version by running a workflow.* +## Expert level + +1. [Custom Delivery](CustomDelivery.md) - *FUTURE TOPIC: Setting up custom delivery to f.ex. a Teams channel.* +1. [Custom Delivery](CustomDeployment.md) - *FUTURE TOPIC: Setting up custom deployment to f.ex. an on-premises environment.* +1. [Custom Jobs](CustomJobs.md) - *FUTURE TOPIC: Adding a custom job to an AL-Go for GitHub workflows.* +1. [Custom Steps](CustomSteps.md) - *FUTURE TOPIC: Adding a custom step to the BuildALGoProject workflow.* +1. [Using Indirect Templates](IndirectTemplates.md) - *FUTURE TOPIC: Using indirect templates to ensure that all repositories are using the same customizations* + ## Additional Future topics 1. Dependencies to other apps diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 new file mode 100644 index 000000000..432e5d9ab --- /dev/null +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -0,0 +1,298 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidGlobalVars', '', Justification = 'Global vars used for local test execution only.')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'All scenario tests have equal parameter set.')] +Param( + [switch] $github, + [switch] $linux, + [string] $githubOwner = $global:E2EgithubOwner, + [string] $repoName = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetTempFileName()), + [string] $token = ($Global:SecureE2EPAT | Get-PlainText), + [string] $pteTemplate = $global:pteTemplate, + [string] $appSourceTemplate = $global:appSourceTemplate, + [string] $adminCenterApiToken = ($global:SecureAdminCenterApiToken | Get-PlainText) +) + +Write-Host -ForegroundColor Yellow @' +# _____ _ _ _ _______ _ _ +# |_ _| | (_) | | |__ __| | | | | +# | | _ __ __| |_ _ __ ___ ___| |_ | | ___ _ __ ___ _ __ | | __ _| |_ ___ +# | | | '_ \ / _` | | '__/ _ \/ __| __| | |/ _ \ '_ ` _ \| '_ \| |/ _` | __/ _ \ +# _| |_| | | | (_| | | | | __/ (__| |_ | | __/ | | | | | |_) | | (_| | || __/ +# |_____|_| |_|\__,_|_|_| \___|\___|\__| |_|\___|_| |_| |_| .__/|_|\__,_|\__\___| +# | | +# |_| +# This test tests the following scenario: +# +# - Create a new repository based on the PTE template with no apps (this will be the "indirect" template repository) +# - Create a new repository based on the PTE template with 1 app, using compilerfolder and donotpublishapps (this will be the "final" template repository) +# - Run Update AL-Go System Files in final repo (using indirect repo as template) +# - Add CustomStep to indirect repo +# - Run Update AL-Go System files in indirect repo +# - Validate that custom step is present in indirect repo +# - Run Update AL-Go System files in final repo +# - Validate that custom step is present in final repo +# - Add CustomStep in final repo +# - Run Update AL-Go System files in final repo +# - Validate that both custom steps is present in final repo +# +'@ + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 +$prevLocation = Get-Location + +Remove-Module e2eTestHelper -ErrorAction SilentlyContinue +Import-Module (Join-Path $PSScriptRoot "..\..\e2eTestHelper.psm1") -DisableNameChecking +. (Join-Path -Path $PSScriptRoot -ChildPath '..\..\..\Actions\CheckForUpdates\yamlclass.ps1') +. (Join-Path -Path $PSScriptRoot -ChildPath "..\..\..\Actions\CheckForUpdates\CheckForUpdates.HelperFunctions.ps1") + +$templateRepository = "$githubOwner/$repoName-template" +$repository = "$githubOwner/$repoName" +$branch = "main" + +$template = "https://github.com/$pteTemplate" + +# Login +SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository + +# Create tempolate repository +CreateAlGoRepository ` + -github:$github ` + -linux:$linux ` + -template $template ` + -repository $templateRepository ` + -branch $branch +$templateRepoPath = (Get-Location).Path + +Set-Location $prevLocation + +$appName = 'MyApp' +$publisherName = 'Contoso' + +# Create repository +CreateAlGoRepository ` + -github:$github ` + -linux:$linux ` + -template $template ` + -repository $repository ` + -branch $branch ` + -contentScript { + Param([string] $path) + $null = CreateNewAppInFolder -folder $path -name $appName -publisher $publisherName + } +$repoPath = (Get-Location).Path + +# Update AL-Go System Files to uptake UseProjectDependencies setting +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $token -repository $repository -branch $branch | Out-Null + +Set-Location $templateRepoPath + +Pull + +# Make modifications to the template repository +$buildALGoProjectWorkflow = Join-Path $templateRepoPath '.github/workflows/_BuildALGoProject.yaml' +$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) +$buildYaml | Should -Not -BeNullOrEmpty + +# Modify the permissions +$buildYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) + +# Add customization steps +$customizationAnchors = GetCustomizationAnchors +$idx = 0 +foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { + $idx++ + $customStep = @{ + "Name" = "CustomStep-Template$idx" + "Content" = @( + "- name: CustomStep-Template$idx" + " run: |" + " Write-Host 'CustomStep-Template$idx was here!'" + ) + "AnchorStep" = $anchor.Step + "Before" = $anchor.Before + } + $buildYaml.AddCustomStepsToAnchor('BuildALGoProject', $customStep, $anchor.Step, $anchor.Before) +} +$buildYaml.Save($buildALGoProjectWorkflow) + +# Add Custom Jobs to CICD.yaml +$cicdWorkflow = Join-Path $templateRepoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty + +# Modify the permissions +$cicdYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) +$customJobs = @( + @{ + "Name" = "CustomJob-TemplateInit" + "Content" = @( + "CustomJob-TemplateInit:" + " runs-on: [ windows-latest ]" + " steps:" + " - name: Init" + " run: |" + " Write-Host 'CustomJob-TemplateInit was here!'" + ) + "NeedsThis" = @( 'Initialization' ) + } + @{ + "Name" = "CustomJob-TemplateDeploy" + "Content" = @( + "CustomJob-TemplateDeploy:" + " needs: [ Initialization, Build ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: Deploy" + " run: |" + " Write-Host 'CustomJob-TemplateDeploy was here!'" + ) + "NeedsThis" = @( 'PostProcess' ) + } +) +# Add custom Jobs +$cicdYaml.AddCustomJobsToYaml($customJobs) +$cicdYaml.Save($cicdWorkflow) + +# Push +CommitAndPush -commitMessage 'Add template customizations' + +# Do not run workflows on template repository +CancelAllWorkflows -repository $templateRepository + +# Add local customizations to the final repository +Set-Location $repoPath +Pull + +# Make modifications to the template repository +$buildALGoProjectWorkflow = Join-Path $repoPath '.github/workflows/_BuildALGoProject.yaml' +$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) +$buildYaml | Should -Not -BeNullOrEmpty + +# Add customization steps +$customizationAnchors = GetCustomizationAnchors +$idx = 0 +foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { + $idx++ + $customStep = @{ + "Name" = "CustomStep-Final$idx" + "Content" = @( + "- name: CustomStep-Final$idx" + " run: |" + " Write-Host 'CustomStep-Final$idx was here!'" + ) + "AnchorStep" = $anchor.Step + "Before" = $anchor.Before + } + $buildYaml.AddCustomStepsToAnchor('BuildALGoProject', $customStep, $anchor.Step, $anchor.Before) +} + +# save +$buildYaml.Save($buildALGoProjectWorkflow) + +# Add Custom Jobs to CICD.yaml +$cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty + +# Modify the permissions +$cicdYaml.Replace('permissions:/contents: read', @('contents: read', 'issues: write')) + +$customJobs = @( + @{ + "Name" = "CustomJob-PreDeploy" + "Content" = @( + "CustomJob-PreDeploy:" + " needs: [ Initialization, Build ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: PreDeploy" + " run: |" + " Write-Host 'CustomJob-PreDeploy was here!'" + ) + "NeedsThis" = @( 'Deploy' ) + } + @{ + "Name" = "CustomJob-PostDeploy" + "Content" = @( + "CustomJob-PostDeploy:" + " needs: [ Initialization, Build, Deploy ]" + " if: (!cancelled())" + " runs-on: [ windows-latest ]" + " steps:" + " - name: PostDeploy" + " run: |" + " Write-Host 'CustomJob-PostDeploy was here!'" + ) + "NeedsThis" = @( 'PostProcess' ) + } +) +# Add custom Jobs +$cicdYaml.AddCustomJobsToYaml($customJobs) + +# save +$cicdYaml.Save($cicdWorkflow) + + +# Push +CommitAndPush -commitMessage 'Add final repo customizations' + +# Update AL-Go System Files to uptake UseProjectDependencies setting +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $token -repository $repository -branch $branch | Out-Null + +# Stop all currently running workflows and run a new CI/CD workflow +CancelAllWorkflows -repository $repository + +# Pull changes +Pull + +# Run CICD +$run = RunCICD -repository $repository -branch $branch -wait + +# Check Custom Steps +1..$idx | ForEach-Object { + Test-LogContainsFromRun -runid $run.id -jobName 'Build . (Default) . (Default)' -stepName "CustomStep-Template$_" -expectedText "CustomStep-Template$_ was here!" +} +1..$idx | ForEach-Object { + Test-LogContainsFromRun -runid $run.id -jobName 'Build . (Default) . (Default)' -stepName "CustomStep-Final$_" -expectedText "CustomStep-Final$_ was here!" +} + +# Check correct order of custom steps +DownloadWorkflowLog -repository $repository -runid $run.id -path 'logs' +$logcontent = Get-Content -Path 'logs/0_Build . (Default) . (Default).txt' -Encoding utf8 -Raw +Remove-Item -Path 'logs' -Recurse -Force +$idx = 0 +foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { + $idx++ + $templateStepIdx = $logcontent.IndexOf("CustomStep-Template$idx was here!") + $finalStepIdx = $logcontent.IndexOf("CustomStep-Final$idx was here!") + if ($anchor.Before) { + $finalStepIdx | Should -BeGreaterThan $templateStepIdx -Because "CustomStep-Final$idx should be after CustomStep-Template$idx" + } + else { + $finalStepIdx | Should -BeLessThan $templateStepIdx -Because "CustomStep-Final$idx should be before CustomStep-Template$idx" + } +} + +# Check Custom Jobs +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateInit' -stepName 'Init' -expectedText 'CustomJob-TemplateInit was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateDeploy' -stepName 'Deploy' -expectedText 'CustomJob-TemplateDeploy was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PreDeploy' -stepName 'PreDeploy' -expectedText 'CustomJob-PreDeploy was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PostDeploy' -stepName 'PostDeploy' -expectedText 'CustomJob-PostDeploy was here!' + +# Check Permissions +# TODO: check issues: write in cicd.yaml (from final) and issues: read in _buildALGoProject.yaml (from template) +$buildALGoProjectWorkflow = Join-Path $repoPath '.github/workflows/_BuildALGoProject.yaml' +$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) +$buildYaml | Should -Not -BeNullOrEmpty +$buildYaml.get('Permissions:/issues:').content | Should -Be 'issues: read' + +$cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty +$cicdYaml.get('Permissions:/issues:').content | Should -Be 'issues: write' + +Set-Location $prevLocation + +Read-Host "Press Enter to continue" + +RemoveRepository -repository $repository -path $repoPath +RemoveRepository -repository $templateRepository -path $templateRepoPath