From 1fede69022fa63709714d14b604712ffca673fd8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 13:29:32 +0100 Subject: [PATCH 001/108] move customizationcap. --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 83 ++++++- Actions/CheckForUpdates/yamlclass.ps1 | 228 ++++++++++++++++++++ 2 files changed, 306 insertions(+), 5 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 0a1f798fd..8bbbb2d0e 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -22,6 +22,18 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper +$anchors = @{ + "_BuildALGoProject.yaml" = @{ + "BuildALGoProject" = @( + @{ "Step" = 'Read settings'; "Before" = $false } + @{ "Step" = 'Read secrets'; "Before" = $false } + @{ "Step" = 'Build'; "Before" = $true } + @{ "Step" = 'Build'; "Before" = $false } + @{ "Step" = 'Cleanup'; "Before" = $true } + ) + } +} + if ($update -eq 'Y') { if (-not $token) { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." @@ -64,19 +76,48 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { $downloadLatest = $true } +$realTemplateFolder = $null $templateFolder = DownloadTemplateRepository -headers $headers -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" $templateBranch = $templateUrl.Split('@')[1] $templateOwner = $templateUrl.Split('/')[3] +$indirectTemplateRepoSettings = @{} +$indirectTemplateProjectSettings = @{} + $isDirectALGo = IsDirectALGo -templateUrl $templateUrl if (-not $isDirectALGo) { $ALGoSettingsFile = Join-Path $templateFolder "*/$repoSettingsFile" if (Test-Path -Path $ALGoSettingsFile -PathType Leaf) { $templateRepoSettings = Get-Content $ALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { - throw "The specified template repository is not a template repository, but instead another AL-Go repository. This is not supported." + # The template repository is a url to another AL-Go repository (an indirect template repository) + # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the indirect template reposotiry points to the "real" template repository + # Copy files and folders from the indirect template repository, but grab the unmodified file from the "real" template repository if it exists and apply customizations + Write-Host "Indirect AL-Go template repository detected, downloading the 'real' template repository" + $realTemplateUrl = $templateRepoSettings.templateUrl + if ($templateRepoSettings.Keys -contains "templateSha") { + $realTemplateSha = $templateRepoSettings.templateSha + } + else { + $realTemplateSha = "" + } + # Download the "real" template repository - use downloadLatest if no TemplateSha is specified in the indirect template repository + $realTemplateFolder = DownloadTemplateRepository -headers $headers -templateUrl $realTemplateUrl -templateSha ([ref]$realTemplateSha) -downloadLatest ($realTemplateSha -eq '') + Write-Host "Real Template Folder: $realTemplateFolder" + + # Set TemplateBranch and TemplateOwner + # Keep TemplateUrl and TemplateSha pointing to the indirect template repository + $templateBranch = $realTemplateUrl.Split('@')[1] + $templateOwner = $realTemplateUrl.Split('/')[3] + + $indirectTemplateRepoSettings = $templateRepoSettings + $projectSettingsFile = Join-Path $templateFolder "*/.AL-Go/settings.json" + if (Test-Path $projectSettingsFile -PathType Leaf) { + Write-Host "Read project settings from indirect template repository" + $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + } } } } @@ -129,9 +170,20 @@ foreach($checkfile in $checkfiles) { $dstPath = $checkfile.dstPath $dstFolder = Join-Path $baseFolder $dstPath $srcFolder = GetSrcFolder -repoSettings $repoSettings -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath + $realSrcFolder = $null + if ($realTemplateFolder) { + $realSrcFolder = GetSrcFolder -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath + } if ($srcFolder) { Push-Location -Path $srcFolder try { + if ($srcPath -eq '.AL-Go' -and $type -eq "script" -and $realSrcFolder) { + Write-Host "Update Project Settings" + # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) + $projectSettingsFile = Join-Path $dstFolder "settings.json" + UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -otherSettings $indirectTemplateProjectSettings + $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } + } # Loop through all files in the template repository matching the pattern Get-ChildItem -Path $srcFolder -Filter $checkfile.pattern | ForEach-Object { # Read the template file and modify it based on the settings @@ -140,24 +192,41 @@ foreach($checkfile in $checkfiles) { Write-Host "- $filename" $dstFile = Join-Path $dstFolder $fileName $srcFile = $_.FullName + $realSrcFile = $srcFile + $isFileDirectALGo = $isDirectALGo Write-Host "SrcFolder: $srcFolder" + if ($realSrcFolder) { + # if SrcFile is an indirect template repository, we need to find the file in the "real" template repository + $fname = Join-Path $realSrcFolder (Resolve-Path $srcFile -Relative) + if (Test-Path -Path $fname -PathType Leaf) { + Write-Host "File is available in the 'real' template repository" + $realSrcFile = $fname + $isFileDirectALGo = IsDirectALGo -templateUrl $realTemplateUrl + } + } if ($type -eq "workflow") { # for workflow files, we might need to modify the file based on the settings - $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $srcFile -repoSettings $repoSettings -depth $depth + $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $realSrcFile -repoSettings $repoSettings -depth $depth } else { # For non-workflow files, just read the file content - $srcContent = Get-ContentLF -Path $srcFile + $srcContent = Get-ContentLF -Path $realSrcFile } # Replace static placeholders $srcContent = $srcContent.Replace('{TEMPLATEURL}', $templateUrl) - if ($isDirectALGo) { + if ($isFileDirectALGo) { # If we are using direct AL-Go repo, we need to change the owner to the remplateOwner, the repo names to AL-Go and AL-Go/Actions and the branch to templateBranch ReplaceOwnerRepoAndBranch -srcContent ([ref]$srcContent) -templateOwner $templateOwner -templateBranch $templateBranch } + if ($type -eq 'workflow' -and $realSrcFile -ne $srcFile) { + # Apply customizations from indirect template repository + Write-Host "Apply customizations from indirect template repository: $srcFile" + [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $anchors) + } + $dstFileExists = Test-Path -Path $dstFile -PathType Leaf if ($unusedALGoSystemFiles -contains $fileName) { # file is not used by ALGo, remove it if it exists @@ -167,6 +236,10 @@ foreach($checkfile in $checkfiles) { } } elseif ($dstFileExists) { + if ($type -eq 'workflow') { + Write-Host "Apply customizations from my repository: $dstFile" + [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $anchors) + } # file exists, compare and add to $updateFiles if different $dstContent = Get-ContentLF -Path $dstFile if ($dstContent -cne $srcContent) { @@ -204,7 +277,7 @@ else { invoke-git status - UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } + UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -additionalSettings $indirectTemplateRepoSettings # Update the files # Calculate the release notes, while updating diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 35987b42a..9fbd894b8 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -126,6 +126,45 @@ class Yaml { return $this.Get($line, [ref] $start, [ref] $count) } + # Locate all lines in the next level of a yaml path + # if $line is empty, you get all first level lines + # Example: + # GetNextLevel("jobs:/") returns @("Initialization:","CheckForUpdates:","Build:","Deploy:",...) + [string[]] GetNextLevel([string] $line) { + [int]$start = 0 + [int]$count = 0 + [Yaml] $yaml = $this + if ($line) { + $yaml = $this.Get($line, [ref] $start, [ref] $count) + } + return $yaml.content | Where-Object { $_ -and -not $_.StartsWith(' ') } + } + + # Get the value of a property as a string + # Example: + # GetProperty("jobs:/Build:/needs:") returns "[ Initialization, Build1 ]" + [string] GetProperty([string] $line) { + [int]$start = 0 + [int]$count = 0 + [Yaml] $yaml = $this.Get($line, [ref] $start, [ref] $count) + if ($yaml -and $yaml.content.Count -eq 1) { + return $yaml.content[0].SubString($yaml.content[0].IndexOf(':')+1).Trim() + } + return $null + } + + # Get the value of a property as a string array + # Example: + # GetPropertyArray("jobs:/Build:/needs:") returns @("Initialization", "Build") + [string[]] GetPropertyArray([string] $line) { + $prop = $this.GetProperty($line) + if ($prop) { + # "needs: [ Initialization, Build ]" becomes @("Initialization", "Build") + return $prop.TrimStart('[').TrimEnd(']').Split(',').Trim() + } + return $null + } + # Replace the lines for the specified Yaml path, given by $line with the lines in $content # If $line ends with '/', then the lines for the section are replaced only # If $line doesn't end with '/', then the line + the lines for the section are replaced @@ -182,4 +221,193 @@ class Yaml { $this.content = $this.content[0..($index-1)] + $yamlContent + $this.content[$index..($this.content.Count-1)] } } + + # Locate jobs in YAML based on a name pattern + # Example: + # GetCustomJobsFromYaml() returns @("CustomJob1", "CustomJob2") + # GetCustomJobsFromYaml("Build*") returns @("Build1","Build2","Build") + [hashtable[]] GetCustomJobsFromYaml([string] $name) { + $result = @() + $allJobs = $this.GetNextLevel('jobs:/').Trim(':') + $customJobs = @($allJobs | Where-Object { $_ -like $name }) + if ($customJobs) { + $nativeJobs = @($allJobs | Where-Object { $customJobs -notcontains $_ }) + Write-Host "Native Jobs:" + foreach($nativeJob in $nativeJobs) { + Write-Host "- $nativeJob" + } + Write-Host "Custom Jobs:" + foreach($customJob in $customJobs) { + Write-Host "- $customJob" + $jobsWithDependency = $nativeJobs | Where-Object { $this.GetPropertyArray("jobs:/$($_):/needs:") | Where-Object { $_ -eq $customJob } } + # If any Build Job has a dependency on this CustomJob, add will be added to all build jobs later + if ($jobsWithDependency | Where-Object { $_ -like 'Build*' }) { + $jobsWithDependency = @($jobsWithDependency | Where-Object { $_ -notlike 'Build*' }) + @('Build') + } + if ($jobsWithDependency) { + Write-Host " - Jobs with dependency: $($jobsWithDependency -join ', ')" + $result += @(@{ "Name" = $customJob; "Content" = @($this.Get("jobs:/$($customJob):").content); "NeedsThis" = @($jobsWithDependency) }) + } + } + } + return $result + } + + # Add jobs to Yaml and update Needs section from native jobs which needs this custom Job + # $customJobs is an array of hashtables with Name, Content and NeedsThis + # Example: + # $customJobs = @(@{ "Name" = "CustomJob1"; "Content" = @(" - pwsh"," -File Build1"); "NeedsThis" = @("Initialization", "Build") }) + # AddCustomJobsToYaml($customJobs) + # The function will add the job CustomJob1 to the Yaml file and update the Needs section of Initialization and Build + # The function will not add the job CustomJob1 if it already exists + [void] AddCustomJobsToYaml([hashtable[]] $customJobs) { + $existingJobs = $this.GetNextLevel('jobs:/').Trim(':') + Write-Host "Adding New Jobs" + foreach($customJob in $customJobs) { + if ($existingJobs -contains $customJob.Name) { + Write-Host "Job $($customJob.Name) already exists" + continue + } + Write-Host "$($customJob.Name) has dependencies from $($customJob.NeedsThis -join ',')" + foreach($needsthis in $customJob.NeedsThis) { + if ($needsthis -eq 'Build') { + $existingJobs | Where-Object { $_ -like 'Build*'} | ForEach-Object { + # Add dependency to all build jobs + $this.Replace("jobs:/$($_):/needs:","needs: [ $(@($this.GetPropertyArray("jobs:/$($_):/needs:"))+@($customJob.Name) -join ', ') ]") + } + } + elseif ($existingJobs -contains $needsthis) { + # Add dependency to job + $this.Replace("jobs:/$($needsthis):/needs:","needs: [ $(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) -join ', ') ]") + } + } + $this.content += @('') + @($customJob.content | ForEach-Object { " $_" }) + } + } + + [string[]] GetStepsFromJob([string] $job) { + $steps = $this.GetNextLevel("Jobs:/$($job):/steps:/") | Where-Object { $_ -like '- name: *' } | ForEach-Object { $_.Substring(8).Trim() } + if ($steps | Group-Object | Where-Object { $_.Count -gt 1 }) { + Write-Host "Duplicate step names in job '$job'" + return @() + } + return $steps + } + + [hashtable[]] GetCustomStepsFromAnchor([string] $job, [string] $anchorStep, [bool] $before) { + $steps = $this.GetStepsFromJob($job) + $anchorIdx = $steps.IndexOf($anchorStep) + if ($anchorIdx -lt 0) { + Write-Host "Cannot find anchor step '$anchorStep' in job '$job'" + return @() + } + $idx = $anchorIdx + $customSteps = @() + if ($before) { + while ($idx -gt 0 -and $steps[$idx-1] -like 'CustomStep*') { + $idx-- + } + if ($idx -ne $anchorIdx) { + $customSteps = @($steps[$idx..($anchorIdx-1)]) + # Reverse the order of the custom steps in order to apply in correct order from the anchor step + [array]::Reverse($customSteps) + } + } + else { + while ($idx -lt $steps.Count-1 -and $steps[$idx+1] -like 'CustomStep*') { + $idx++ + } + if ($idx -ne $anchorIdx) { + $customSteps = @($steps[($anchorIdx+1)..$idx]) + } + } + $result = @() + foreach($customStep in $customSteps) { + $stepContent = $this.Get("Jobs:/$($job):/steps:/- name: $($customStep)").content + $result += @(@{"Name" = $customStep; "Content" = $stepContent; "AnchorStep" = $anchorStep; "Before" = $before }) + } + return $result + } + + [hashtable[]] GetCustomStepsFromYaml([string] $job, [hashtable[]] $anchors) { + $steps = $this.GetStepsFromJob($job) + $result = @() + foreach($anchor in $anchors) { + $result += $this.GetCustomStepsFromAnchor($job, $anchor.Step, $anchor.Before) + } + foreach($step in $steps) { + if ($step -like 'CustomStep*') { + if (-not ($result | Where-Object { $_.Name -eq $step })) { + Write-Host "Custom step '$step' does not belong to a supported anchor" + } + } + } + return $result + } + + [void] AddCustomStepsToAnchor([string] $job, [hashtable[]] $customSteps, [string] $anchorStep, [bool] $before) { + $steps = $this.GetStepsFromJob($job) + if (!$steps) { + Write-Host "::Warning::Cannot find job '$job'" + return + } + $anchorIdx = $steps.IndexOf($anchorStep) + if ($anchorIdx -lt 0) { + Write-Host "::Warning::Cannot find anchor step '$anchorStep' in job '$job'" + return + } + foreach($customStep in $customSteps | Where-Object { $_.AnchorStep -eq $anchorStep -and $_.Before -eq $before }) { + if ($steps -contains $customStep.Name) { + Write-Host "Custom step '$($customStep.Name)' already exists in job '$job'" + } + else { + $anchorStart = 0 + $anchorCount = 0 + if ($this.Find("Jobs:/$($job):/steps:/- name: $($anchorStep)", [ref] $anchorStart, [ref] $anchorCount)) { + if ($before) { + $this.Insert($anchorStart-1, @('') + @($customStep.Content | ForEach-Object { " $_" })) + } + else { + $this.Insert($anchorStart+$anchorCount, @('') + @($customStep.Content | ForEach-Object { " $_" })) + } + } + } + # Use added step as anchor for next step + $anchorStep = $customStep.Name + } + } + + [void] AddCustomStepsToYaml([string] $job, [hashtable[]] $customSteps, [hashtable[]] $anchors) { + foreach($anchor in $anchors) { + $this.AddCustomStepsToAnchor($job, $customSteps, $anchor.Step, $anchor.Before) + } + } + + static [void] ApplyCustomizations([ref] $srcContent, [string] $yamlFile, [hashtable] $anchors) { + $srcYaml = [Yaml]::new($srcContent.Value.Split("`n")) + try { + $yaml = [Yaml]::Load($yamlFile) + } + catch { + return + } + $filename = [System.IO.Path]::GetFileName($yamlFile) + if ($anchors.ContainsKey($filename)) { + $fileAnchors = $anchors."$filename" + foreach($job in $fileAnchors.Keys) { + # Locate custom steps in destination YAML + $customSteps = $yaml.GetCustomStepsFromYaml($job, $fileAnchors."$job") + if ($customSteps) { + $srcYaml.AddCustomStepsToYaml($job, $customSteps, $fileAnchors."$job") + } + } + } + # Locate custom jobs in destination YAML + $customJobs = @($yaml.GetCustomJobsFromYaml('CustomJob*')) + if ($customJobs) { + # Add custom jobs to template YAML + $srcYaml.AddCustomJobsToYaml($customJobs) + } + $srcContent.Value = $srcYaml.content -join "`n" + } } From ad207daa28b623c7d0851eaf86ba49ccc355966c Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 13:45:08 +0100 Subject: [PATCH 002/108] use type --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 4 ++-- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 492e06735..b7872dbd4 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -279,7 +279,7 @@ function IsDirectALGo { function GetSrcFolder { Param( - [hashtable] $repoSettings, + [string] $repoType, [string] $templateUrl, [string] $templateFolder, [string] $srcPath @@ -291,7 +291,7 @@ function GetSrcFolder { return '' } if (IsDirectALGo -templateUrl $templateUrl) { - switch ($repoSettings.type) { + switch ($repoType) { "PTE" { $typePath = "Per Tenant Extension" } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 8bbbb2d0e..51de4f5f0 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -169,10 +169,10 @@ foreach($checkfile in $checkfiles) { $srcPath = $checkfile.srcPath $dstPath = $checkfile.dstPath $dstFolder = Join-Path $baseFolder $dstPath - $srcFolder = GetSrcFolder -repoSettings $repoSettings -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath + $srcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath $realSrcFolder = $null if ($realTemplateFolder) { - $realSrcFolder = GetSrcFolder -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath + $realSrcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $realTemplateUrl -templateFolder $realTemplateFolder -srcPath $srcPath } if ($srcFolder) { Push-Location -Path $srcFolder From 2c0e1c2f312b897abe9a260e291deb7801834a9c Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 13:56:57 +0100 Subject: [PATCH 003/108] use additional --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 51de4f5f0..ad478d7cb 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,7 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - $projectSettingsFile = Join-Path $templateFolder "*/.AL-Go/settings.json" + $projectSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" if (Test-Path $projectSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse @@ -181,7 +181,7 @@ foreach($checkfile in $checkfiles) { Write-Host "Update Project Settings" # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) $projectSettingsFile = Join-Path $dstFolder "settings.json" - UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -otherSettings $indirectTemplateProjectSettings + UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -additionalSettings $indirectTemplateProjectSettings $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } } # Loop through all files in the template repository matching the pattern From e9fc84793863e95f90110ef2ce51d4ec01f90142 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:05:50 +0100 Subject: [PATCH 004/108] realtemplate --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index ad478d7cb..60d8cfd32 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,7 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - $projectSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" + $projectSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" if (Test-Path $projectSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse From cb81cfa5204738e86115c74d80dd347b3c3fbbb8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:08:49 +0100 Subject: [PATCH 005/108] dump --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 60d8cfd32..09727aef9 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,9 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings + Write-Host $ALGoSettingsFile $projectSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" + Write-Host $projectSettingsFile if (Test-Path $projectSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse From 7c3046ba3700161a9b1efa2c2db6681d23a4f6cc Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:11:34 +0100 Subject: [PATCH 006/108] use correct settings --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 09727aef9..1b99de7f5 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -88,9 +88,9 @@ $indirectTemplateProjectSettings = @{} $isDirectALGo = IsDirectALGo -templateUrl $templateUrl if (-not $isDirectALGo) { - $ALGoSettingsFile = Join-Path $templateFolder "*/$repoSettingsFile" - if (Test-Path -Path $ALGoSettingsFile -PathType Leaf) { - $templateRepoSettings = Get-Content $ALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + $myRepoSettingsFile = Join-Path $templateFolder "*/$RepoSettingsFile" + if (Test-Path -Path $myRepoSettingsFile -PathType Leaf) { + $templateRepoSettings = Get-Content $myRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { # The template repository is a url to another AL-Go repository (an indirect template repository) # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the indirect template reposotiry points to the "real" template repository @@ -113,12 +113,10 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - Write-Host $ALGoSettingsFile - $projectSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" - Write-Host $projectSettingsFile - if (Test-Path $projectSettingsFile -PathType Leaf) { + $myALGoSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" + if (Test-Path $myALGoSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" - $indirectTemplateProjectSettings = Get-Content $projectSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + $indirectTemplateProjectSettings = Get-Content $myALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse } } } From cfe76e859dfa47b4d8d98bab97879f8c72c92acf Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:14:45 +0100 Subject: [PATCH 007/108] use correct template --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 1b99de7f5..b02216bd8 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,7 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - $myALGoSettingsFile = Join-Path $realTemplateFolder "*/$ALGoSettingsFile" + $myALGoSettingsFile = Join-Path $templateFolder $ALGoSettingsFile if (Test-Path $myALGoSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $myALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse From 753c7fc81d6a7f54575074dab0c41d7ce2b9a0ef Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 14:17:27 +0100 Subject: [PATCH 008/108] add subfolder --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index b02216bd8..d8c558184 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -113,7 +113,7 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings - $myALGoSettingsFile = Join-Path $templateFolder $ALGoSettingsFile + $myALGoSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" if (Test-Path $myALGoSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" $indirectTemplateProjectSettings = Get-Content $myALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse From f03dbd3e4329e31448431a26f198cd93c35ea2c2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 22:57:23 +0100 Subject: [PATCH 009/108] release notes --- RELEASENOTES.md | 17 ++++++++++++++++- Scenarios/settings.md | 6 ++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 8a8e5141b..cb81e48d8 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -13,6 +13,21 @@ Note that when using the preview version of AL-Go for GitHub, we recommend you U Add another parameter when running Update AL-Go System Files, called downloadLatest, used to indicate whether to download latest version from template repository. Default value is true. If false, the templateSha repository setting is used to download specific AL-Go System Files when calculating new files. +### Add custom jobs to AL-Go workflows +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go system files. + +### Add custom steps to the AL-Go build workflow +It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. + +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go system files. + +### Indirect AL-Go template repository +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + ### Issues - Issue 782 Exclude '.altestrunner/' from template .gitignore - Issue 823 Dependencies from prior build jobs are not included when using useProjectDependencies @@ -22,7 +37,7 @@ If false, the templateSha repository setting is used to download specific AL-Go ### New Settings -- `alDoc` : JSON object with properties for the ALDoc reference document generation +- `[alDoc](https://aka.ms/algosettings#aldoc)` : JSON object with properties for the ALDoc reference document generation - **continuousDeployment** = Determines if reference documentation will be deployed continuously as part of CI/CD. You can run the **Deploy Reference Documentation** workflow to deploy manually or on a schedule. (Default false) - **deployToGitHubPages** = Determines whether or not the reference documentation site should be deployed to GitHub Pages for the repository. In order to deploy to GitHub Pages, GitHub Pages must be enabled and set to GitHub Actions. (Default true) - **maxReleases** = Maximum number of releases to include in the reference documentation. (Default 3) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index c83b4262b..f462981de 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -294,6 +294,12 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | [ ] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | +## Custom jobs in AL-Go for GitHub workflows + +## Custom steps in the _BuildALGoProject workflow + +## Indirect template repositories + ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. From c5ee954924ae94e9605ef3c867dd9e4bcf5fba3d Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 22:58:35 +0100 Subject: [PATCH 010/108] lf --- Scenarios/settings.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index f462981de..b38163a29 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -294,11 +294,14 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | [ ] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | -## Custom jobs in AL-Go for GitHub workflows + +## Custom jobs in AL-Go for GitHub workflows -## Custom steps in the _BuildALGoProject workflow + +## Custom steps in the _BuildALGoProject workflow -## Indirect template repositories + +## Indirect template repositories ## Your own version of AL-Go for GitHub From 811cc63cb3487778109d27383f69ceca111bf576 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 22:59:49 +0100 Subject: [PATCH 011/108] lines --- Scenarios/settings.md | 127 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index b38163a29..19a0784fe 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -297,12 +297,139 @@ Settings, which might be relevant to set in the settings file includes ## Custom jobs in AL-Go for GitHub workflows +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + ## Custom steps in the _BuildALGoProject workflow +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + + ## Indirect template repositories +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + +test1 + +test2 + + + ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. From 5971157d686bb2505fb554dce4675c81958746e3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 12 Dec 2023 23:07:18 +0100 Subject: [PATCH 012/108] exp --- Scenarios/settings.md | 158 ++++++------------------------------------ 1 file changed, 20 insertions(+), 138 deletions(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 19a0784fe..bd5764e15 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -21,6 +21,7 @@ When running a workflow or a local script, the settings are applied by reading s 1. `.AL-Go/.settings.json` is the **user-specific settings file**. This option is rarely used, but if you have special settings, which should only be used for one specific user (potentially in the local scripts), these settings can be added to a settings file with the name of the user followed by `.settings.json`. + ## Basic settings | Name | Description | Default value | @@ -68,6 +69,7 @@ The repository settings are only read from the repository settings file (.github | UpdateGitHubGoSystemFilesSchedule | CRON schedule for when Update AL-Go System Files should run. When Update AL-Go System Files runs on a schedule, it uses direct Commit instead of creating a PR. Default is no scheduled run, only manual trigger. Build your CRON string here: [https://crontab.guru](https://crontab.guru) | | buildModes | A list of build modes to use when building the AL-Go projects. Every AL-Go projects will be built using each built mode. Available build modes are:
**Default**: Apps are compiled as they are in the source code.
**Clean**: _PreprocessorSymbols_ are enabled when compiling the apps. The values for the symbols correspond to the `cleanModePreprocessorSymbols` setting of the AL-Go project.
**Translated**: `TranslationFile` compiler feature is enabled when compiling the apps. | + ## Advanced settings | Name | Description | Default value | @@ -115,6 +117,7 @@ The repository settings are only read from the repository settings file (.github | appSourceContextSecretName | This setting specifies the name (**NOT the secret**) of a secret containing a json string with ClientID, TenantID and ClientSecret or RefreshToken. If this secret exists, AL-Go will can upload builds to AppSource validation. | AppSourceContext | | keyVaultCertificateUrlSecretName
keyVaultCertificatePasswordSecretName
keyVaultClientIdSecretName | If you want to enable KeyVault access for your AppSource App, you need to provide 3 secrets as GitHub Secrets or in the Azure KeyVault. The names of those secrets (**NOT the secrets**) should be specified in the settings file with these 3 settings. Default is to not have KeyVault access from your AppSource App. Read [this](EnableKeyVaultForAppSourceApp.md) for more information. | | + ## Conditional Settings In any of the settings files, you can add conditional settings by using the ConditionalSettings setting. @@ -161,6 +164,11 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza **Note:** that you can have conditional settings on any level and all conditional settings which has all conditions met will be applied in the order of settings file + appearance. + +# Expert level + +The following settings and functionality requires knowledge in GitHub Workflows/Actions and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. + ## Expert settings (rarely used) | Name | Description | Default value | @@ -181,8 +189,7 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | [ ] | -# Expert level - + ## Custom Delivery You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named DeliverTo*.ps1 in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. @@ -222,6 +229,7 @@ Here are the parameters to use in your custom script: | `$parameters.testAppsFolders` | The folders that contain the build artifacts from all builds (from different build modes) of the test apps in the AL-Go project | AllProjects_MyProject-main-TestApps-1.0.0.0, AllProjects_MyProject-main-CleanTestApps-1.0.0.0 | | `$parameters.dependenciesFolders` | The folders that contain the dependencies of the AL-Go project for all builds (from different build modes) | AllProjects_MyProject-main-Dependencies-1.0.0.0, AllProjects_MyProject-main-CleanDependencies-1.0.0.0 | + ## Custom Deployment You can override existing AL-Go Deployment functionality or you can define your own custom deployment mechanism for AL-Go for GitHub. By specifying a PowerShell script named `DeployTo.ps1` in the .github folder. Default Environment Type is SaaS, but you can define your own type by specifying EnvironmentType in the `DeployTo` setting. The following example will spin up a deployment job to SharePoint on CI/CD and Publish To Environment. @@ -255,6 +263,16 @@ Here are the parameters to use in your custom script: | `$parameters.ContinuousDeployment` | Is this environment setup for continuous deployment | false | | `$parameters."runs-on"` | GitHub runner to be used to run the deployment script | windows-latest | + + +## Custom jobs in AL-Go for GitHub workflows + + +## Custom steps in the _BuildALGoProject workflow + + +## Indirect template repositories + ## Run-AlPipeline script override AL-Go for GitHub utilizes the Run-AlPipeline function from BcContainerHelper to perform the actual build (compile, publish, test etc). The Run-AlPipeline function supports overriding functions for creating containers, compiling apps and a lot of other things. @@ -294,142 +312,6 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | [ ] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | - -## Custom jobs in AL-Go for GitHub workflows - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - - -## Custom steps in the _BuildALGoProject workflow - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - - - -## Indirect template repositories - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - -test1 - -test2 - - - ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. From 8fe0008be796ddf522fbe97c95c515b778d6530f Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 13 Dec 2023 06:38:34 +0100 Subject: [PATCH 013/108] docs --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- RELEASENOTES.md | 4 +- Scenarios/settings.md | 97 ++++++++++++++++++--- Workshop/Index.md | 7 ++ 4 files changed, 95 insertions(+), 15 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index d8c558184..05fd2b9ab 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -226,7 +226,7 @@ foreach($checkfile in $checkfiles) { Write-Host "Apply customizations from indirect template repository: $srcFile" [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $anchors) } - + $dstFileExists = Test-Path -Path $dstFile -PathType Leaf if ($unusedALGoSystemFiles -contains $fileName) { # file is not used by ALGo, remove it if it exists diff --git a/RELEASENOTES.md b/RELEASENOTES.md index cb81e48d8..1c777dcf3 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -16,12 +16,12 @@ If false, the templateSha repository setting is used to download specific AL-Go ### Add custom jobs to AL-Go workflows It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go system files. +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. ### Add custom steps to the AL-Go build workflow It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. -Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go system files. +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. ### Indirect AL-Go template repository Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index bd5764e15..c78b044ab 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -167,7 +167,9 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza # Expert level -The following settings and functionality requires knowledge in GitHub Workflows/Actions and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. +The settings and functionality in the expert section might requires knowledge about GitHub Workflows/Actions, yaml, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. + +Please read the release notes carefully when installing new versions of AL-Go for GitHub. ## Expert settings (rarely used) @@ -263,22 +265,14 @@ Here are the parameters to use in your custom script: | `$parameters.ContinuousDeployment` | Is this environment setup for continuous deployment | false | | `$parameters."runs-on"` | GitHub runner to be used to run the deployment script | windows-latest | - - -## Custom jobs in AL-Go for GitHub workflows - - -## Custom steps in the _BuildALGoProject workflow - - -## Indirect template repositories - ## Run-AlPipeline script override AL-Go for GitHub utilizes the Run-AlPipeline function from BcContainerHelper to perform the actual build (compile, publish, test etc). The Run-AlPipeline function supports overriding functions for creating containers, compiling apps and a lot of other things. This functionality is also available in AL-Go for GitHub, by adding a file to the .AL-Go folder, you automatically override the function. +Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the future might break the usage of these overrides. + | Override | Description | | :-- | :-- | | DockerPull.ps1 | Pull the image specified by the parameter $imageName | @@ -302,6 +296,8 @@ The repo settings file (.github\\AL-Go-Settings.json) can contain BcContainerHel Settings, which might be relevant to set in the settings file includes +Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the future might break the usage of these overrides. + | Setting | Description | Default | | :-- | :-- | :-- | | baseUrl | The Base Url for the online Business Central Web Client. This should be changed when targetting embed apps. | [https://businesscentral.dynamics.com](https://businesscentral.dynamics.com) | @@ -312,11 +308,88 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | [ ] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | + +## Custom jobs in AL-Go for GitHub workflows + +Adding a custom job to any AL-Go for GitHub workflow is done by adding a job with the name `CustomJob` to the end of an AL-Go for GitHub workflow, like this: + +``` + CustomJob-PrepareDeploy: + name: My Job + needs: [ Build ] + runs-on: [ ubuntu-latest ] + defaults: + run: + shell: pwsh + steps: + - name: This is my job + run: | + Write-Host "This is my job" +``` + +In the `needs` property, you specify which jobs should be complete before this job is run. If you require this job to run before other AL-Go for GitHub jobs are complete, you can add the name of this job in the `needs` property of that job, like: + +``` + Deploy: + needs: [ Initialization, Build, **CustomJob-PrepareDeploy** ] + if: always() && needs.Build.result == 'Success' && needs.Initialization.outputs.environmentCount > 0 + strategy: ${{ fromJson(needs.Initialization.outputs.environmentsMatrixJson) }} +``` + +Custom jobs will be preserved when running Update AL-Go System Files. + +**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). + + +## Custom steps in the _BuildALGoProject workflow + +Adding a custom step is done by adding a step with the name `CustomStep` to the _BuildALGoProject.yaml workflow at one of these anchor points: +- Before Read Settings +- Before Read Secrets +- Before or After Build +- Before Cleanup + +Example, insert the following step before the Build step: + +``` + - name: CustomStep that will run before the Build step + run: | + Write-Host "before build" + + - name: Build + uses: ... +``` + +Custom steps will be preserved when running Update AL-Go System Files. + +**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). + + +## Indirect template repositories + +If you are utilizing script overrides, custom jobs, custom steps, custom delivery or like in many repositories, you might want to take advantage of the indirect template repository feature. + +An indirect template repository is an AL-Go for GitHub repository (without any apps), which is used as a template for the remaining AL-Go for GitHub repositories. As an example, if you are using a custom delivery script, which you want to have in all your repositories, you can create an empty AL-Go for GitHub repository, place the delivery script in the .github folder and use that repository as a template when running Update AL-Go system files in your other repositories. + +This would make sure that all repositories would have this script (and updated versions of the script) in the future. + +The items, which are currently supported from indirect template repositories are: +- Repository script overrides in the .github folder +- Project script overrides in the .AL-Go folder +- Custom workflows in the .github/workflows folder +- Custom jobs in any AL-Go for GitHub workflow +- Custom steps in the _BuildALGoProject workflow +- New repository settings +- New project settings + +**Note** that an AL-Go for GitHub indirect template repository can be private or public. + ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. -**Note:** Our goal is to never break repositories, which are using AL-Go for GitHub as their template. We almost certainly will break you if you create local modifications to scripts and pipelines. +> [!NOTE] +> Our goal is to never break repositories, which are using standard AL-Go for GitHub as their template. We almost certainly will break you if you create local modifications to scripts and pipelines. --- [back](../README.md) diff --git a/Workshop/Index.md b/Workshop/Index.md index 717334675..fbfc8fb44 100644 --- a/Workshop/Index.md +++ b/Workshop/Index.md @@ -21,6 +21,13 @@ This workshop shows you how to take advantage of the functionality, which is pro 1. [The Development Process](TheDevelopmentProcess.md) - *FUTURE TOPIC: The recommended way to work with feature branches, pull requests, code reviews and branch protection rules.* 1. [Keeping your Repository Up-to-date](KeepUpToDate.md) - *FUTURE TOPIC: Updating AL-Go for GitHub to the latest version by running a workflow.* +## Expert level + +1. [Custom Delivery](CustomDelivery.md) - *FUTURE TOPIC: Setting up custom delivery to f.ex. a Teams channel.* +1. [Custom Delivery](CustomDeployment.md) - *FUTURE TOPIC: Setting up custom deployment to f.ex. an on-premises environment.* +1. [Custom Jobs](CustomJobs.md) - *FUTURE TOPIC: Adding a custom job to an AL-Go for GitHub workflows.* +1. [Custom Steps](CustomSteps.md) - *FUTURE TOPIC: Adding a custom step to the BuildALGoProject workflow.* +1. [Using Indirect Templates](IndirectTemplates.md) - *FUTURE TOPIC: Using indirect templates to ensure that all repositories are using the same customizations* ## Additional Future topics 1. Dependencies to other apps From 32b5586c292378a5f60023d937f5fc81557c6535 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 13 Dec 2023 06:39:39 +0100 Subject: [PATCH 014/108] aspit --- Scenarios/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index c78b044ab..f22ebb23c 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -389,7 +389,7 @@ The items, which are currently supported from indirect template repositories are For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. > [!NOTE] -> Our goal is to never break repositories, which are using standard AL-Go for GitHub as their template. We almost certainly will break you if you create local modifications to scripts and pipelines. +> Our goal is to never break repositories, which are using standard AL-Go for GitHub as their template. We almost certainly will break you at some point in time if you create local modifications to scripts and pipelines. --- [back](../README.md) From 2bcc4ae09f921c3df085f60381b676145b4245f5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 13 Dec 2023 06:40:13 +0100 Subject: [PATCH 015/108] remove bold --- Scenarios/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index f22ebb23c..76bdcf284 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -331,7 +331,7 @@ In the `needs` property, you specify which jobs should be complete before this j ``` Deploy: - needs: [ Initialization, Build, **CustomJob-PrepareDeploy** ] + needs: [ Initialization, Build, CustomJob-PrepareDeploy ] if: always() && needs.Build.result == 'Success' && needs.Initialization.outputs.environmentCount > 0 strategy: ${{ fromJson(needs.Initialization.outputs.environmentsMatrixJson) }} ``` From 3c6f7cd1c88124f0ed292768bf725566faf64ba9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 15 Dec 2023 22:16:06 +0100 Subject: [PATCH 016/108] do not use AL-Go name in package name --- Actions/Deliver/Deliver.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/Actions/Deliver/Deliver.ps1 b/Actions/Deliver/Deliver.ps1 index b10aac177..a4ebefddd 100644 --- a/Actions/Deliver/Deliver.ps1 +++ b/Actions/Deliver/Deliver.ps1 @@ -218,9 +218,6 @@ try { Get-Item -Path (Join-Path $folder[0] "*.app") | ForEach-Object { $parameters = @{ "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY" - "includeNuGetDependencies" = $true - "dependencyIdTemplate" = "AL-Go-{id}" - "packageId" = "AL-Go-{id}" } $parameters.appFiles = $_.FullName $package = New-BcNuGetPackage @parameters From 418d40f6786b8c91bffa32d0c4693d704ac31ee4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 24 Jun 2024 19:38:25 +0200 Subject: [PATCH 017/108] pre-commit --- RELEASENOTES.md | 33 ++++++++++++++++++--------------- Scenarios/settings.md | 17 ++++++++++++++--- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 93071e0a6..fa2419b82 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -17,6 +17,24 @@ In the summary after a Test Run, you now also have the result of performance tes > \[!NOTE\] > Duration thresholds are subject to varying results depending on the performance of the agent running the tests. Number of SQL statements executed by a test is often the most reliable indicator of performance degredation. +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Add custom steps to the AL-Go build workflow + +It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. + +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Indirect AL-Go template repository + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + ## v5.2 ### Issues @@ -137,21 +155,6 @@ AL-Go ships with Default, Translated and Clean mode out of the box. Now you can Add another parameter when running Update AL-Go System Files, called downloadLatest, used to indicate whether to download latest version from template repository. Default value is true. If false, the templateSha repository setting is used to download specific AL-Go System Files when calculating new files. -### Add custom jobs to AL-Go workflows -It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. - -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Add custom steps to the AL-Go build workflow -It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. - -Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Indirect AL-Go template repository -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. - -Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. - ### Issues - Issue 782 Exclude '.altestrunner/' from template .gitignore diff --git a/Scenarios/settings.md b/Scenarios/settings.md index f6fe68d7f..7aac0c392 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -23,6 +23,7 @@ When running a workflow or a local script, the settings are applied by reading s 1. `.AL-Go/.settings.json` is the **user-specific settings file**. This option is rarely used, but if you have special settings, which should only be used for one specific user (potentially in the local scripts), these settings can be added to a settings file with the name of the user followed by `.settings.json`. + ## Basic Project settings | Name | Description | Default value | @@ -78,6 +79,7 @@ The repository settings are only read from the repository settings file (.github | buildModes | A list of build modes to use when building the AL-Go projects. Every AL-Go project will be built using each build mode. AL-Go ships with the following build modes out of the box:
**Default**: Apps are compiled as they are in the source code.
**Clean**: _PreprocessorSymbols_ are enabled when compiling the apps. The values for the symbols correspond to the `cleanModePreprocessorSymbols` setting of the AL-Go project.
**Translated**: `TranslationFile` compiler feature is enabled when compiling the apps.

It is also possible to specify custom build modes by adding a build mode that is different than 'Default', 'Clean' or 'Translated'. | + ## Advanced settings | Name | Description | Default value | @@ -127,6 +129,7 @@ The repository settings are only read from the repository settings file (.github | keyVaultCertificateUrlSecretName
keyVaultCertificatePasswordSecretName
keyVaultClientIdSecretName | If you want to enable KeyVault access for your AppSource App, you need to provide 3 secrets as GitHub Secrets or in the Azure KeyVault. The names of those secrets (**NOT the secrets**) should be specified in the settings file with these 3 settings. Default is to not have KeyVault access from your AppSource App. Read [this](EnableKeyVaultForAppSourceApp.md) for more information. | | + ## Conditional Settings In any of the settings files, you can add conditional settings by using the ConditionalSettings setting. @@ -179,6 +182,7 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza > You can have conditional settings on any level and all conditional settings which has all conditions met will be applied in the order of settings file + appearance. + # Expert level The settings and functionality in the expert section might requires knowledge about GitHub Workflows/Actions, yaml, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. @@ -206,6 +210,7 @@ Please read the release notes carefully when installing new versions of AL-Go fo | unusedALGoSystemFiles | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | \[ \] | + ## Custom Delivery You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named DeliverTo\*.ps1 in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. @@ -248,6 +253,7 @@ Here are the parameters to use in your custom script: | `$parameters.dependenciesFolders` | The folders that contain the dependencies of the AL-Go project for all builds (from different build modes) | AllProjects_MyProject-main-Dependencies-1.0.0.0, AllProjects_MyProject-main-CleanDependencies-1.0.0.0 | + ## Custom Deployment You can override existing AL-Go Deployment functionality or you can define your own custom deployment mechanism for AL-Go for GitHub. By specifying a PowerShell script named `DeployTo.ps1` in the .github folder. Default Environment Type is SaaS, but you can define your own type by specifying EnvironmentType in the `DeployTo` setting. The following example will spin up a deployment job to SharePoint on CI/CD and Publish To Environment. @@ -328,6 +334,7 @@ Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the fut | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | + ## Custom jobs in AL-Go for GitHub workflows Adding a custom job to any AL-Go for GitHub workflow is done by adding a job with the name `CustomJob` to the end of an AL-Go for GitHub workflow, like this: @@ -360,9 +367,11 @@ Custom jobs will be preserved when running Update AL-Go System Files. **Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). -## Custom steps in the _BuildALGoProject workflow -Adding a custom step is done by adding a step with the name `CustomStep` to the _BuildALGoProject.yaml workflow at one of these anchor points: +## Custom steps in the \_BuildALGoProject workflow + +Adding a custom step is done by adding a step with the name `CustomStep` to the \_BuildALGoProject.yaml workflow at one of these anchor points: + - Before Read Settings - Before Read Secrets - Before or After Build @@ -384,6 +393,7 @@ Custom steps will be preserved when running Update AL-Go System Files. **Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs and custom steps, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). + ## Indirect template repositories If you are utilizing script overrides, custom jobs, custom steps, custom delivery or like in many repositories, you might want to take advantage of the indirect template repository feature. @@ -393,11 +403,12 @@ An indirect template repository is an AL-Go for GitHub repository (without any a This would make sure that all repositories would have this script (and updated versions of the script) in the future. The items, which are currently supported from indirect template repositories are: + - Repository script overrides in the .github folder - Project script overrides in the .AL-Go folder - Custom workflows in the .github/workflows folder - Custom jobs in any AL-Go for GitHub workflow -- Custom steps in the _BuildALGoProject workflow +- Custom steps in the \_BuildALGoProject workflow - New repository settings - New project settings From 2efb6a5929f8e32c29c989c00cc5956446777ce0 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 24 Jul 2024 12:29:34 +0200 Subject: [PATCH 018/108] back --- Actions/Deliver/Deliver.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Actions/Deliver/Deliver.ps1 b/Actions/Deliver/Deliver.ps1 index 76c3d52fe..0d22b793d 100644 --- a/Actions/Deliver/Deliver.ps1 +++ b/Actions/Deliver/Deliver.ps1 @@ -246,6 +246,9 @@ try { Get-Item -Path (Join-Path $folder[0] "*.app") | ForEach-Object { $parameters = @{ "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY" + "includeNuGetDependencies" = $true + "dependencyIdTemplate" = "AL-Go-{id}" + "packageId" = "AL-Go-{id}" } $parameters.appFiles = $_.FullName $package = New-BcNuGetPackage @parameters From bcc2f3e2a9eae55dba8bf63803dd65d43e34f461 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 11:13:41 +0200 Subject: [PATCH 019/108] add permissions --- Actions/CheckForUpdates/yamlclass.ps1 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 69f1ebb76..331837edb 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -394,6 +394,15 @@ class Yaml { catch { return } + $srcPermissions = $srcYaml.GetPropertyArray('permissions:') + $yamlPermissions = $yaml.GetPropertyArray('permissions:') + if ($srcPermissions) { + $yamlPermissions | ForEach-Object { + if ($srcPermissions -notcontains $_) { + $srcPermissions.Add($_) + } + } + } $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { $fileAnchors = $anchors."$filename" From 2c62b84f167c754aee0abcc6a72a99b221c44742 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 11:24:48 +0200 Subject: [PATCH 020/108] add --- Actions/CheckForUpdates/yamlclass.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 331837edb..537d22401 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -394,8 +394,8 @@ class Yaml { catch { return } - $srcPermissions = $srcYaml.GetPropertyArray('permissions:') - $yamlPermissions = $yaml.GetPropertyArray('permissions:') + $srcPermissions = $srcYaml.Get('permissions:/') + $yamlPermissions = $yaml.Get('permissions:/') if ($srcPermissions) { $yamlPermissions | ForEach-Object { if ($srcPermissions -notcontains $_) { From 496db9104f15ebd713faea7555d706bfa684948b Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 11:29:16 +0200 Subject: [PATCH 021/108] add --- Actions/CheckForUpdates/yamlclass.ps1 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 537d22401..b190113d3 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -225,6 +225,14 @@ class Yaml { } } + # Add lines to Yaml content + [void] Add([string[]] $yamlContent) { + if (!$yamlContent) { + return + } + $this.Insert($this.content.Count, $yamlContent) + } + # Locate jobs in YAML based on a name pattern # Example: # GetCustomJobsFromYaml() returns @("CustomJob1", "CustomJob2") @@ -394,6 +402,7 @@ class Yaml { catch { return } + # Merge permissions $srcPermissions = $srcYaml.Get('permissions:/') $yamlPermissions = $yaml.Get('permissions:/') if ($srcPermissions) { From 4445277b86f72a4cd05442743b02b2856cdfb83d Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 11:48:26 +0200 Subject: [PATCH 022/108] dumps --- Actions/CheckForUpdates/yamlclass.ps1 | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index b190113d3..a85f62671 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -403,15 +403,25 @@ class Yaml { return } # Merge permissions + Write-host "Merge permissions" $srcPermissions = $srcYaml.Get('permissions:/') $yamlPermissions = $yaml.Get('permissions:/') + Write-Host "yamlPermissions:" + $yamlPermissions.content | Out-Host + Write-Host "srcPermissions:" + $srcPermissions.content | Out-Host if ($srcPermissions) { - $yamlPermissions | ForEach-Object { - if ($srcPermissions -notcontains $_) { + $yamlPermissions.content | ForEach-Object { + Write-Host $_ + if ($srcPermissions.content -notcontains $_) { + Write-Host "Add permission $_" $srcPermissions.Add($_) } } } + Write-Host "srcPermissions (after):" + $srcPermissions.content | Out-Host + $srcYaml.Replace('permissions:/', $srcPermissions.content) $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { $fileAnchors = $anchors."$filename" From 7ef5d3d186c7f5d5776bad1f3a0944f32d95a588 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:05:02 +0200 Subject: [PATCH 023/108] add merge --- Actions/CheckForUpdates/yamlclass.ps1 | 59 +++++++++++++++++++-------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index a85f62671..52839ad50 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -394,6 +394,44 @@ class Yaml { } } + static [hashtable] GetPermissionsFromArray([string[]] $permissionsArray) { + $permissions = @{} + $permissionsArray | ForEach-Object { + $permissions += @{ $_.Split(':')[0].Trim() = $_.Split(':')[1].Trim() } + } + return $permissions + } + + static [string[]] GetPermissionsArray([hashtable] $permissions) { + $permissionsArray = @() + $permissions.Keys | ForEach-Object { + $permissionsArray += "$($_): $($permissions[$_])" + } + return $permissionsArray + } + + static [hashtable] MergePermissions([hashtable] $permissions, [hashtable] $permissions2) { + $permissions2.Keys | ForEach-Object { + if ($permissions.ContainsKey($_)) { + $permission = $permissions[$_] + $permission2 = $permissions2[$_] + if ($permission -eq 'Write' -or $permission2 -eq 'Write') { + $permissions[$_] = 'Write' + } + elseif ($permission -eq 'Read' -or $permission2 -eq 'Read') { + $permissions[$_] = 'Read' + } + else { + $permissions[$_] = 'None' + } + } + else { + $permissions += @{ $_ = $permissions2[$_] } + } + } + return $permissions + } + static [void] ApplyCustomizations([ref] $srcContent, [string] $yamlFile, [hashtable] $anchors) { $srcYaml = [Yaml]::new($srcContent.Value.Split("`n")) try { @@ -404,24 +442,9 @@ class Yaml { } # Merge permissions Write-host "Merge permissions" - $srcPermissions = $srcYaml.Get('permissions:/') - $yamlPermissions = $yaml.Get('permissions:/') - Write-Host "yamlPermissions:" - $yamlPermissions.content | Out-Host - Write-Host "srcPermissions:" - $srcPermissions.content | Out-Host - if ($srcPermissions) { - $yamlPermissions.content | ForEach-Object { - Write-Host $_ - if ($srcPermissions.content -notcontains $_) { - Write-Host "Add permission $_" - $srcPermissions.Add($_) - } - } - } - Write-Host "srcPermissions (after):" - $srcPermissions.content | Out-Host - $srcYaml.Replace('permissions:/', $srcPermissions.content) + $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) + $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) + $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { $fileAnchors = $anchors."$filename" From 6e8b95cbd33b109755b1e56537418ce7fd6804d0 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:05:54 +0200 Subject: [PATCH 024/108] merge --- Actions/CheckForUpdates/yamlclass.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 52839ad50..b56bd2ed7 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,6 +445,7 @@ class Yaml { $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + Write-Host "Apply custom steps" $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { $fileAnchors = $anchors."$filename" @@ -457,6 +458,7 @@ class Yaml { } } # Locate custom jobs in destination YAML + Write-Host "Apply custom jobs" $customJobs = @($yaml.GetCustomJobsFromYaml('CustomJob*')) if ($customJobs) { # Add custom jobs to template YAML From 422386d50aab33206368a860084bebf9312d34b8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:14:42 +0200 Subject: [PATCH 025/108] use pso --- Actions/CheckForUpdates/yamlclass.ps1 | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index b56bd2ed7..59cd119b9 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -394,39 +394,39 @@ class Yaml { } } - static [hashtable] GetPermissionsFromArray([string[]] $permissionsArray) { - $permissions = @{} + static [PSCustomObject] GetPermissionsFromArray([string[]] $permissionsArray) { + $permissions = [PSCustomObject]@{} $permissionsArray | ForEach-Object { - $permissions += @{ $_.Split(':')[0].Trim() = $_.Split(':')[1].Trim() } + $permissions | Add-Member -MemberType NoteProperty -Name $_.Split(':')[0].Trim() -Value $_.Split(':')[1].Trim() } return $permissions } - static [string[]] GetPermissionsArray([hashtable] $permissions) { + static [string[]] GetPermissionsArray([PSCustomObject] $permissions) { $permissionsArray = @() - $permissions.Keys | ForEach-Object { + $permissions.PSObject.Properties.Name | ForEach-Object { $permissionsArray += "$($_): $($permissions[$_])" } return $permissionsArray } - static [hashtable] MergePermissions([hashtable] $permissions, [hashtable] $permissions2) { - $permissions2.Keys | ForEach-Object { - if ($permissions.ContainsKey($_)) { + static [PSCustomObject] MergePermissions([PSCustomObject] $permissions, [PSCustomObject] $permissions2) { + $permissions2.PSObject.Properties.Name | ForEach-Object { + if ($permissions.PSObject.Properties.Name -eq $_) { $permission = $permissions[$_] $permission2 = $permissions2[$_] if ($permission -eq 'Write' -or $permission2 -eq 'Write') { - $permissions[$_] = 'Write' + $permissions."$_" = 'Write' } elseif ($permission -eq 'Read' -or $permission2 -eq 'Read') { - $permissions[$_] = 'Read' + $permissions."$_" = 'Read' } else { - $permissions[$_] = 'None' + $permissions."$_" = 'None' } } else { - $permissions += @{ $_ = $permissions2[$_] } + $permissions | Add-Member -MemberType NoteProperty -Name $_ -Value $permissions2[$_] } } return $permissions From 40e20fd6864f8215cf0f7f58dd3321b90d8ff672 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:16:41 +0200 Subject: [PATCH 026/108] use prop --- Actions/CheckForUpdates/yamlclass.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 59cd119b9..7e6fb9dba 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -413,8 +413,8 @@ class Yaml { static [PSCustomObject] MergePermissions([PSCustomObject] $permissions, [PSCustomObject] $permissions2) { $permissions2.PSObject.Properties.Name | ForEach-Object { if ($permissions.PSObject.Properties.Name -eq $_) { - $permission = $permissions[$_] - $permission2 = $permissions2[$_] + $permission = $permissions."$_" + $permission2 = $permissions2."$_" if ($permission -eq 'Write' -or $permission2 -eq 'Write') { $permissions."$_" = 'Write' } From 846626e3e92b4c9d62ccdc40d429f71052042e3c Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:17:05 +0200 Subject: [PATCH 027/108] test --- .../.github/workflows/_BuildALGoProject.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index 214e500ad..9b5a2dc6f 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -73,7 +73,7 @@ on: type: string permissions: - contents: read + contents: write actions: read id-token: write From 82361a814f13f775fdab45ebfaf814bc5b00470c Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:18:26 +0200 Subject: [PATCH 028/108] no idx --- Actions/CheckForUpdates/yamlclass.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 7e6fb9dba..049c48bc1 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -405,7 +405,7 @@ class Yaml { static [string[]] GetPermissionsArray([PSCustomObject] $permissions) { $permissionsArray = @() $permissions.PSObject.Properties.Name | ForEach-Object { - $permissionsArray += "$($_): $($permissions[$_])" + $permissionsArray += "$($_): $($permissions."$_")" } return $permissionsArray } From 09362823e445f22fd937ac367b7bb309d1e517e9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:20:07 +0200 Subject: [PATCH 029/108] no idx --- Actions/CheckForUpdates/yamlclass.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 049c48bc1..f05ccbe38 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -426,7 +426,7 @@ class Yaml { } } else { - $permissions | Add-Member -MemberType NoteProperty -Name $_ -Value $permissions2[$_] + $permissions | Add-Member -MemberType NoteProperty -Name $_ -Value $permissions2."$_" } } return $permissions From b1566937b758c3c36f045bb868e7aab5c3e18601 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 12:22:42 +0200 Subject: [PATCH 030/108] lower case --- Actions/CheckForUpdates/yamlclass.ps1 | 10 +++++----- .../.github/workflows/_BuildALGoProject.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index f05ccbe38..dbce79590 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -415,14 +415,14 @@ class Yaml { if ($permissions.PSObject.Properties.Name -eq $_) { $permission = $permissions."$_" $permission2 = $permissions2."$_" - if ($permission -eq 'Write' -or $permission2 -eq 'Write') { - $permissions."$_" = 'Write' + if ($permission -eq 'write' -or $permission2 -eq 'write') { + $permissions."$_" = 'write' } - elseif ($permission -eq 'Read' -or $permission2 -eq 'Read') { - $permissions."$_" = 'Read' + elseif ($permission -eq 'read' -or $permission2 -eq 'read') { + $permissions."$_" = 'read' } else { - $permissions."$_" = 'None' + $permissions."$_" = 'none' } } else { diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index 9b5a2dc6f..214e500ad 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -73,7 +73,7 @@ on: type: string permissions: - contents: write + contents: read actions: read id-token: write From af146b731832634e88134ee990aca0f3e687b27d Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 25 Jul 2024 18:39:46 +0200 Subject: [PATCH 031/108] add permissions --- Scenarios/Contribute.md | 1 - .../.github/workflows/_BuildPowerPlatformSolution.yaml | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Scenarios/Contribute.md b/Scenarios/Contribute.md index cd385fa8b..8ff23d670 100644 --- a/Scenarios/Contribute.md +++ b/Scenarios/Contribute.md @@ -80,7 +80,6 @@ You can also run the end to end tests directly from VS Code, by providing the fo |$global:E2EgitHubOwner| String | The GitHub owner of the test repositories (like `freddydk` or `microsoft`) | |$global:SecureE2EPAT| SecureString | A personal access token with workflow permissions | |$global:SecureAdminCenterApiToken| SecureString | Admin Center API Credentials | -|$global:SecureLicenseFileUrl| SecureString | Direct download URL to a license file | |$global:pteTemplate| String | URL for your PTE template (like `freddyk/AL-Go-PTE@main` or `freddydk/AL-Go@main\|Templates/Per Tenant Extension` for using your AL-Go fork directly) | |$global:appSourceTemplate| String | URL for your PTE template (like `freddyk/AL-Go-AppSource@main` or `freddydk/AL-Go@main\|Templates/AppSource App` for using your AL-Go fork directly) | diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml index 314365f96..28dfbd066 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml @@ -41,6 +41,10 @@ on: required: false type: string +permissions: + contents: read + actions: read + env: ALGoOrgSettings: ${{ vars.ALGoOrgSettings }} ALGoRepoSettings: ${{ vars.ALGoRepoSettings }} From b9219b78ec33d912ec41c9d325ea791a153e24c3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 27 Jul 2024 09:24:17 +0200 Subject: [PATCH 032/108] updates --- .../CheckForUpdates.HelperFunctions.ps1 | 14 ++++++++++++++ Actions/CheckForUpdates/CheckForUpdates.ps1 | 17 +++-------------- Actions/CheckForUpdates/yamlclass.ps1 | 2 ++ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 560a7be49..466993bea 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -1,3 +1,17 @@ +function GetCustomizationAnchors { + return @{ + "_BuildALGoProject.yaml" = @{ + "BuildALGoProject" = @( + @{ "Step" = 'Read settings'; "Before" = $false } + @{ "Step" = 'Read secrets'; "Before" = $false } + @{ "Step" = 'Build'; "Before" = $true } + @{ "Step" = 'Build'; "Before" = $false } + @{ "Step" = 'Cleanup'; "Before" = $true } + ) + } + } +} + <# .SYNOPSIS Downloads a template repository and returns the path to the downloaded folder diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index a9683250b..1d84a6c40 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -22,18 +22,6 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper -$anchors = @{ - "_BuildALGoProject.yaml" = @{ - "BuildALGoProject" = @( - @{ "Step" = 'Read settings'; "Before" = $false } - @{ "Step" = 'Read secrets'; "Before" = $false } - @{ "Step" = 'Build'; "Before" = $true } - @{ "Step" = 'Build'; "Before" = $false } - @{ "Step" = 'Cleanup'; "Before" = $true } - ) - } -} - if ($update -eq 'Y') { if (-not $token) { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." @@ -227,10 +215,11 @@ foreach($checkfile in $checkfiles) { ReplaceOwnerRepoAndBranch -srcContent ([ref]$srcContent) -templateOwner $templateOwner -templateBranch $templateBranch } + $customizationAnchors = GetCustomizationAnchors if ($type -eq 'workflow' -and $realSrcFile -ne $srcFile) { # Apply customizations from indirect template repository Write-Host "Apply customizations from indirect template repository: $srcFile" - [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $anchors) + [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $customizationAnchors) } $dstFileExists = Test-Path -Path $dstFile -PathType Leaf @@ -244,7 +233,7 @@ foreach($checkfile in $checkfiles) { elseif ($dstFileExists) { if ($type -eq 'workflow') { Write-Host "Apply customizations from my repository: $dstFile" - [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $anchors) + [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $customizationAnchors) } # file exists, compare and add to $updateFiles if different $dstContent = Get-ContentLF -Path $dstFile diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index dbce79590..f3b405b1a 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,6 +445,8 @@ class Yaml { $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + + # Apply cystom steps Write-Host "Apply custom steps" $filename = [System.IO.Path]::GetFileName($yamlFile) if ($anchors.ContainsKey($filename)) { From dd36934a6f1a589ae41e72e34249957f3b7eb3a4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 27 Jul 2024 09:42:58 +0200 Subject: [PATCH 033/108] content parameter --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- Actions/CheckForUpdates/yamlclass.ps1 | 2 +- Actions/CreateApp/AppHelper.psm1 | 2 +- Actions/Github-Helper.psm1 | 9 ++++++--- e2eTests/e2eTestHelper.psm1 | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 1d84a6c40..b9b3dd6d8 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -308,7 +308,7 @@ else { } } Write-Host "Update $($_.DstFile)" - $_.Content | Set-ContentLF -Path $_.DstFile + Set-ContentLF -Path $_.DstFile -Content $_.Content } if ($releaseNotes -eq "") { $releaseNotes = "No release notes available!" diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index f3b405b1a..908b20926 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -20,7 +20,7 @@ class Yaml { # Save the Yaml file with LF line endings using UTF8 encoding Save([string] $filename) { - $this.content | Set-ContentLF -Path $filename + Set-ContentLF -Path $filename -Content $this.content } # Find the lines for the specified Yaml path, given by $line diff --git a/Actions/CreateApp/AppHelper.psm1 b/Actions/CreateApp/AppHelper.psm1 index 19221df37..bcb3ad8e6 100644 --- a/Actions/CreateApp/AppHelper.psm1 +++ b/Actions/CreateApp/AppHelper.psm1 @@ -97,7 +97,7 @@ function UpdateALFile $al = $al.Replace("$_", $startId) $startId++ } - $al | Set-ContentLF -Path (Join-Path $destinationFolder $alFileName) + Set-ContentLF -Path (Join-Path $destinationFolder $alFileName) -Content $al } <# diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index c4439d7f4..ecd5f5926 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -715,12 +715,15 @@ function Set-ContentLF { Process { $path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($path) if ($content -is [array]) { - $content = $content -join "`n" + Write-Host "array" + $contentStr = $content -join "`n" + Write-Host $contentStr } else { - $content = "$content".Replace("`r", "") + Write-Host "String" + $contentStr = "$content".Replace("`r", "") } - [System.IO.File]::WriteAllText($path, "$content`n") + [System.IO.File]::WriteAllText($path, "$contentStr`n") } } diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index 9900c4958..a79048c9a 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -337,7 +337,7 @@ function CreateNewAppInFolder { $folder = Join-Path $folder $name New-Item -Path $folder -ItemType Directory | Out-Null $appJson | Set-JsonContentLF -Path (Join-Path $folder "app.json") - $al -join "`n" | Set-ContentLF -Path (Join-Path $folder "$name.al") + Set-ContentLF -Path (Join-Path $folder "$name.al") -Content $al $id } From f6c2dfb66673449be31f37ed23081a3cc832f37f Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 27 Jul 2024 09:50:51 +0200 Subject: [PATCH 034/108] use str --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- Actions/CheckForUpdates/yamlclass.ps1 | 2 +- Actions/CreateApp/AppHelper.psm1 | 2 +- Actions/Github-Helper.psm1 | 14 +++----------- e2eTests/e2eTestHelper.psm1 | 2 +- 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index b9b3dd6d8..1d84a6c40 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -308,7 +308,7 @@ else { } } Write-Host "Update $($_.DstFile)" - Set-ContentLF -Path $_.DstFile -Content $_.Content + $_.Content | Set-ContentLF -Path $_.DstFile } if ($releaseNotes -eq "") { $releaseNotes = "No release notes available!" diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 908b20926..676ac8a4c 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -20,7 +20,7 @@ class Yaml { # Save the Yaml file with LF line endings using UTF8 encoding Save([string] $filename) { - Set-ContentLF -Path $filename -Content $this.content + $this.content -join "`n" | Set-ContentLF -Path $filename } # Find the lines for the specified Yaml path, given by $line diff --git a/Actions/CreateApp/AppHelper.psm1 b/Actions/CreateApp/AppHelper.psm1 index bcb3ad8e6..19221df37 100644 --- a/Actions/CreateApp/AppHelper.psm1 +++ b/Actions/CreateApp/AppHelper.psm1 @@ -97,7 +97,7 @@ function UpdateALFile $al = $al.Replace("$_", $startId) $startId++ } - Set-ContentLF -Path (Join-Path $destinationFolder $alFileName) -Content $al + $al | Set-ContentLF -Path (Join-Path $destinationFolder $alFileName) } <# diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index ecd5f5926..399b8073d 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -709,21 +709,13 @@ function Set-ContentLF { [parameter(mandatory = $true, ValueFromPipeline = $false)] [string] $path, [parameter(mandatory = $true, ValueFromPipeline = $true)] - $content + [string] $content ) Process { $path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($path) - if ($content -is [array]) { - Write-Host "array" - $contentStr = $content -join "`n" - Write-Host $contentStr - } - else { - Write-Host "String" - $contentStr = "$content".Replace("`r", "") - } - [System.IO.File]::WriteAllText($path, "$contentStr`n") + $content = "$content".Replace("`r", "").TrimEnd("`n") + [System.IO.File]::WriteAllText($path, "$content`n") } } diff --git a/e2eTests/e2eTestHelper.psm1 b/e2eTests/e2eTestHelper.psm1 index a79048c9a..9900c4958 100644 --- a/e2eTests/e2eTestHelper.psm1 +++ b/e2eTests/e2eTestHelper.psm1 @@ -337,7 +337,7 @@ function CreateNewAppInFolder { $folder = Join-Path $folder $name New-Item -Path $folder -ItemType Directory | Out-Null $appJson | Set-JsonContentLF -Path (Join-Path $folder "app.json") - Set-ContentLF -Path (Join-Path $folder "$name.al") -Content $al + $al -join "`n" | Set-ContentLF -Path (Join-Path $folder "$name.al") $id } From 2992605e7551c145040857740bf2e63b85478841 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 27 Jul 2024 10:32:50 +0200 Subject: [PATCH 035/108] fix needs --- Actions/CheckForUpdates/yamlclass.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 676ac8a4c..7f463ed47 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -289,7 +289,8 @@ class Yaml { } elseif ($existingJobs -contains $needsthis) { # Add dependency to job - $this.Replace("jobs:/$($needsthis):/needs:","needs: [ $(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) -join ', ') ]") + $needs = @(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) | Where-Object { $_ }) -join ', ' + $this.Replace("jobs:/$($needsthis):/needs:","needs: [ $needs ]") } } $this.content += @('') + @($customJob.content | ForEach-Object { " $_" }) From b3534d50ef7b86560aa6dd64626ca849747dff45 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 30 Jul 2024 09:31:59 +0200 Subject: [PATCH 036/108] add end 2 end test scenario and unit tests --- Tests/CheckForUpdates.Action.Test.ps1 | 29 ++ Tests/CustomizedYamlSnippet.txt | 97 ++++++ .../scenarios/IndirectTemplate/runtest.ps1 | 296 ++++++++++++++++++ 3 files changed, 422 insertions(+) create mode 100644 Tests/CustomizedYamlSnippet.txt create mode 100644 e2eTests/scenarios/IndirectTemplate/runtest.ps1 diff --git a/Tests/CheckForUpdates.Action.Test.ps1 b/Tests/CheckForUpdates.Action.Test.ps1 index ea3cb62b7..6fff8f3a2 100644 --- a/Tests/CheckForUpdates.Action.Test.ps1 +++ b/Tests/CheckForUpdates.Action.Test.ps1 @@ -121,5 +121,34 @@ Describe "CheckForUpdates Action Tests" { $permissionsContent.content[1].Trim() | Should -be 'actions: read' } + It 'Test YamlClass Customizations' { + . (Join-Path $scriptRoot "yamlclass.ps1") + + $customizedYaml = [Yaml]::load((Join-Path $PSScriptRoot 'CustomizedYamlSnippet.txt')) + $yaml = [Yaml]::load((Join-Path $PSScriptRoot 'YamlSnippet.txt')) + + # Get Custom jobs from yaml + $customJobs = $customizedYaml.GetCustomJobsFromYaml('CustomJob*') + $customJobs | Should -Not -BeNullOrEmpty + $customJobs.Count | Should -be 1 + + # Get Custom steps from yaml + $customStep1 = $customizedYaml.GetCustomStepsFromAnchor('Initialization', 'Read settings', $true) + $customStep1 | Should -Not -BeNullOrEmpty + $customStep1.Count | Should -be 2 + + $customStep2 = $customizedYaml.GetCustomStepsFromAnchor('Initialization', 'Read settings', $false) + $customStep2 | Should -Not -BeNullOrEmpty + $customStep2.Count | Should -be 1 + + # Apply Custom jobs and steps to yaml + $yaml.AddCustomJobsToYaml($customJobs) + $yaml.AddCustomStepsToAnchor('Initialization', $customStep1, 'Read settings', $true) + $yaml.AddCustomStepsToAnchor('Initialization', $customStep2, 'Read settings', $false) + + # Check if new yaml content is equal to customized yaml content + ($yaml.content -join "`r`n") | Should -be ($customizedYaml.content -join "`r`n") + } + # Call action } diff --git a/Tests/CustomizedYamlSnippet.txt b/Tests/CustomizedYamlSnippet.txt new file mode 100644 index 000000000..ae1b5ba46 --- /dev/null +++ b/Tests/CustomizedYamlSnippet.txt @@ -0,0 +1,97 @@ +name: 'CI/CD' + +on: + workflow_dispatch: + workflow_run: + workflows: ["Pull Request Handler"] + types: + - completed + push: + paths-ignore: + - '**.md' + - '.github/workflows/*.yaml' + - '!.github/workflows/CICD.yaml' + branches: [ 'main', 'release/*', 'feature/*' ] + +run-name: ${{ fromJson(format('["","Check pull request from {1}/{2}{0} {3}"]',':',github.event.workflow_run.head_repository.owner.login,github.event.workflow_run.head_branch,github.event.workflow_run.display_title))[github.event_name == 'workflow_run'] }} + +permissions: + contents: read + actions: read + pull-requests: write + checks: write + +defaults: + run: + shell: powershell + +env: + workflowDepth: 1 + +jobs: + Initialization: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + runs-on: [ windows-latest ] + outputs: + telemetryScopeJson: ${{ steps.init.outputs.telemetryScopeJson }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: true + + - name: Initialize the workflow + id: init + uses: microsoft/AL-Go-Actions/WorkflowInitialize@main + with: + shell: powershell + eventId: "DO0091" + + - name: CustomStep-MyStep1 + run: | + Write-Host 'My own step1!' + + - name: CustomStep-MyStepX + run: | + Write-Host 'Some stepX!' + + - name: Read settings + id: ReadSettings + uses: microsoft/AL-Go-Actions/ReadSettings@main + with: + shell: powershell + parentTelemetryScopeJson: ${{ steps.init.outputs.telemetryScopeJson }} + + - name: CustomStep-MyStep2 + run: | + Write-Host 'My own step2!' + + CheckForUpdates: + runs-on: [ windows-latest ] + needs: [ Initialization, CustomJob-MyJob ] + if: github.event_name != 'workflow_run' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read settings + uses: microsoft/AL-Go-Actions/ReadSettings@main + with: + shell: powershell + parentTelemetryScopeJson: ${{ needs.Initialization.outputs.telemetryScopeJson }} + get: templateUrl + + - name: Check for updates to AL-Go system files + uses: microsoft/AL-Go-Actions/CheckForUpdates@main + with: + shell: powershell + parentTelemetryScopeJson: ${{ needs.Initialization.outputs.telemetryScopeJson }} + templateUrl: ${{ env.templateUrl }} + + CustomJob-MyJob: + needs: [ Initialization ] + runs-on: [ windows-latest ] + steps: + - name: MyStep + run: | + Write-Host 'My own job!' diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 new file mode 100644 index 000000000..101d1862a --- /dev/null +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -0,0 +1,296 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidGlobalVars', '', Justification = 'Global vars used for local test execution only.')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'All scenario tests have equal parameter set.')] +Param( + [switch] $github, + [switch] $linux, + [string] $githubOwner = $global:E2EgithubOwner, + [string] $repoName = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetTempFileName()), + [string] $token = ($Global:SecureE2EPAT | Get-PlainText), + [string] $pteTemplate = $global:pteTemplate, + [string] $appSourceTemplate = $global:appSourceTemplate, + [string] $adminCenterApiToken = ($global:SecureAdminCenterApiToken | Get-PlainText) +) + +Write-Host -ForegroundColor Yellow @' +# _____ _ _ _ _______ _ _ +# |_ _| | (_) | | |__ __| | | | | +# | | _ __ __| |_ _ __ ___ ___| |_ | | ___ _ __ ___ _ __ | | __ _| |_ ___ +# | | | '_ \ / _` | | '__/ _ \/ __| __| | |/ _ \ '_ ` _ \| '_ \| |/ _` | __/ _ \ +# _| |_| | | | (_| | | | | __/ (__| |_ | | __/ | | | | | |_) | | (_| | || __/ +# |_____|_| |_|\__,_|_|_| \___|\___|\__| |_|\___|_| |_| |_| .__/|_|\__,_|\__\___| +# | | +# |_| +# This test tests the following scenario: +# +# - Create a new repository based on the PTE template with no apps (this will be the "indirect" template repository) +# - Create a new repository based on the PTE template with 1 app, using compilerfolder and donotpublishapps (this will be the "final" template repository) +# - Run Update AL-Go System Files in final repo (using indirect repo as template) +# - Add CustomStep to indirect repo +# - Run Update AL-Go System files in indirect repo +# - Validate that custom step is present in indirect repo +# - Run Update AL-Go System files in final repo +# - Validate that custom step is present in final repo +# - Add CustomStep in final repo +# - Run Update AL-Go System files in final repo +# - Validate that both custom steps is present in final repo +# +'@ + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 +$prevLocation = Get-Location + +Remove-Module e2eTestHelper -ErrorAction SilentlyContinue +Import-Module (Join-Path $PSScriptRoot "..\..\e2eTestHelper.psm1") -DisableNameChecking +. (Join-Path -Path $PSScriptRoot -ChildPath '..\..\..\Actions\CheckForUpdates\yamlclass.ps1') +. (Join-Path -Path $PSScriptRoot -ChildPath "..\..\..\Actions\CheckForUpdates\CheckForUpdates.HelperFunctions.ps1") + +$templateRepository = "$githubOwner/$repoName-template" +$repository = "$githubOwner/$repoName" +$branch = "main" + +$template = "https://github.com/$pteTemplate" + +# Login +SetTokenAndRepository -github:$github -githubOwner $githubOwner -token $token -repository $repository + +# Create tempolate repository +CreateAlGoRepository ` + -github:$github ` + -linux:$linux ` + -template $template ` + -repository $templateRepository ` + -branch $branch +$templateRepoPath = (Get-Location).Path + +Set-Location $prevLocation + +$appName = 'MyApp' +$publisherName = 'Contoso' + +# Create repository +CreateAlGoRepository ` + -github:$github ` + -linux:$linux ` + -template $template ` + -repository $repository ` + -branch $branch ` + -contentScript { + Param([string] $path) + $null = CreateNewAppInFolder -folder $path -name $appName -publisher $publisherName + } +$repoPath = (Get-Location).Path + +# Update AL-Go System Files to uptake UseProjectDependencies setting +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $token -repository $repository -branch $branch | Out-Null + +Set-Location $templateRepoPath + +Pull + +# Make modifications to the template repository +$buildALGoProjectWorkflow = Join-Path $templateRepoPath '.github/workflows/_BuildALGoProject.yaml' +$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) +$buildYaml | Should -Not -BeNullOrEmpty + +# Modify the permissions +$buildYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) + +# Add customization steps +$customizationAnchors = GetCustomizationAnchors +$idx = 0 +foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { + $idx++ + $customStep = @{ + "Name" = "CustomStep-Template$idx" + "Content" = @( + "- name: CustomStep-Template$idx" + " run: |" + " Write-Host 'CustomStep-Template$idx was here!'" + ) + "AnchorStep" = $anchor.Step + "Before" = $anchor.Before + } + $buildYaml.AddCustomStepsToAnchor('BuildALGoProject', $customStep, $anchor.Step, $anchor.Before) +} +$buildYaml.Save($buildALGoProjectWorkflow) + +# Add Custom Jobs to CICD.yaml +$cicdWorkflow = Join-Path $templateRepoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty +# Modify the permissions +$cicdYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) +$customJobs = @( + @{ + "Name" = "CustomJob-TemplateInit" + "Content" = @( + "CustomJob-TemplateInit:" + " runs-on: [ windows-latest ]" + " steps:" + " - name: Init" + " run: |" + " Write-Host 'CustomJob-TemplateInit was here!'" + ) + "NeedsThis" = @( 'Initialization' ) + } + @{ + "Name" = "CustomJob-TemplateDeploy" + "Content" = @( + "CustomJob-TemplateDeploy:" + " needs: [ Initialization, Build ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: Deploy" + " run: |" + " Write-Host 'CustomJob-TemplateDeploy was here!'" + ) + "NeedsThis" = @( 'PostProcess' ) + } +) +# Add custom Jobs +$cicdYaml.AddCustomJobsToYaml($customJobs) +$cicdYaml.Save($cicdWorkflow) + +# Push +CommitAndPush -commitMessage 'Add template customizations' + +# Do not run workflows on template repository +CancelAllWorkflows -repository $templateRepository + +# Add local customizations to the final repository +Set-Location $repoPath +Pull + +# Make modifications to the template repository +$buildALGoProjectWorkflow = Join-Path $repoPath '.github/workflows/_BuildALGoProject.yaml' +$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) +$buildYaml | Should -Not -BeNullOrEmpty + +# Add customization steps +$customizationAnchors = GetCustomizationAnchors +$idx = 0 +foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { + $idx++ + $customStep = @{ + "Name" = "CustomStep-Final$idx" + "Content" = @( + "- name: CustomStep-Final$idx" + " run: |" + " Write-Host 'CustomStep-Final$idx was here!'" + ) + "AnchorStep" = $anchor.Step + "Before" = $anchor.Before + } + $buildYaml.AddCustomStepsToAnchor('BuildALGoProject', $customStep, $anchor.Step, $anchor.Before) +} + +# save +$buildYaml.Save($buildALGoProjectWorkflow) + +# Add Custom Jobs to CICD.yaml +$cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty +# Modify the permissions +$cicdYaml.Replace('permissions:/contents: read', @('contents: read', 'issues: write')) + +$customJobs = @( + @{ + "Name" = "CustomJob-PreDeploy" + "Content" = @( + "CustomJob-PreDeploy:" + " needs: [ Initialization, Build ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: PreDeploy" + " run: |" + " Write-Host 'CustomJob-PreDeploy was here!'" + ) + "NeedsThis" = @( 'Deploy' ) + } + @{ + "Name" = "CustomJob-PostDeploy" + "Content" = @( + "CustomJob-PostDeploy:" + " needs: [ Initialization, Build, Deploy ]" + " if: (!cancelled())" + " runs-on: [ windows-latest ]" + " steps:" + " - name: PostDeploy" + " run: |" + " Write-Host 'CustomJob-PostDeploy was here!'" + ) + "NeedsThis" = @( 'PostProcess' ) + } +) +# Add custom Jobs +$cicdYaml.AddCustomJobsToYaml($customJobs) + +# save +$cicdYaml.Save($cicdWorkflow) + + +# Push +CommitAndPush -commitMessage 'Add final repo customizations' + +# Update AL-Go System Files to uptake UseProjectDependencies setting +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $token -repository $repository -branch $branch | Out-Null + +# Stop all currently running workflows and run a new CI/CD workflow +CancelAllWorkflows -repository $repository + +# Pull changes +Pull + +# Run CICD +$run = RunCICD -repository $repository -branch $branch -wait + +# Check Custom Steps +1..$idx | ForEach-Object { + Test-LogContainsFromRun -runid $run.id -jobName 'Build . (Default) . (Default)' -stepName "CustomStep-Template$_" -expectedText "CustomStep-Template$_ was here!" +} +1..$idx | ForEach-Object { + Test-LogContainsFromRun -runid $run.id -jobName 'Build . (Default) . (Default)' -stepName "CustomStep-Final$_" -expectedText "CustomStep-Final$_ was here!" +} + +# Check correct order of custom steps +DownloadWorkflowLog -repository $repository -runid $run.id -path 'logs' +$logcontent = Get-Content -Path 'logs/0_Build . (Default) . (Default).txt' -Encoding utf8 -Raw +Remove-Item -Path 'logs' -Recurse -Force +$idx = 0 +foreach($anchor in $customizationAnchors.'_BuildALGoProject.yaml'.BuildALGoProject) { + $idx++ + $templateStepIdx = $logcontent.IndexOf("CustomStep-Template$idx was here!") + $finalStepIdx = $logcontent.IndexOf("CustomStep-Final$idx was here!") + if ($anchor.Before) { + $finalStepIdx | Should -BeGreaterThan $templateStepIdx -Because "CustomStep-Final$idx should be after CustomStep-Template$idx" + } + else { + $finalStepIdx | Should -BeLessThan $templateStepIdx -Because "CustomStep-Final$idx should be before CustomStep-Template$idx" + } +} + +# Check Custom Jobs +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateInit' -stepName 'Init' -expectedText 'CustomJob-TemplateInit was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateDeploy' -stepName 'Deploy' -expectedText 'CustomJob-TemplateDeploy was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PreDeploy' -stepName 'PreDeploy' -expectedText 'CustomJob-PreDeploy was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PostDeploy' -stepName 'PostDeploy' -expectedText 'CustomJob-PostDeploy was here!' + +# Check Permissions +# TODO: check issues: write in cicd.yaml (from final) and issues: read in _buildALGoProject.yaml (from template) +$buildALGoProjectWorkflow = Join-Path $repoPath '.github/workflows/_BuildALGoProject.yaml' +$buildYaml = [yaml]::Load($buildALGoProjectWorkflow) +$buildYaml | Should -Not -BeNullOrEmpty +$buildYaml.get('Permissions:/issues:').content | Should -Be 'issues: read' + +$cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty +$cicdYaml.get('Permissions:/issues:').content | Should -Be 'issues: write' + +Set-Location $prevLocation + +Read-Host "Press Enter to continue" + +RemoveRepository -repository $repository -path $repoPath +RemoveRepository -repository $templateRepository -path $templateRepoPath From 5f34aede27741a15e215072c13576edc30896460 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 30 Jul 2024 09:55:55 +0200 Subject: [PATCH 037/108] precommit --- .../scenarios/IndirectTemplate/runtest.ps1 | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 index 101d1862a..432e5d9ab 100644 --- a/e2eTests/scenarios/IndirectTemplate/runtest.ps1 +++ b/e2eTests/scenarios/IndirectTemplate/runtest.ps1 @@ -12,14 +12,14 @@ Param( ) Write-Host -ForegroundColor Yellow @' -# _____ _ _ _ _______ _ _ -# |_ _| | (_) | | |__ __| | | | | -# | | _ __ __| |_ _ __ ___ ___| |_ | | ___ _ __ ___ _ __ | | __ _| |_ ___ +# _____ _ _ _ _______ _ _ +# |_ _| | (_) | | |__ __| | | | | +# | | _ __ __| |_ _ __ ___ ___| |_ | | ___ _ __ ___ _ __ | | __ _| |_ ___ # | | | '_ \ / _` | | '__/ _ \/ __| __| | |/ _ \ '_ ` _ \| '_ \| |/ _` | __/ _ \ # _| |_| | | | (_| | | | | __/ (__| |_ | | __/ | | | | | |_) | | (_| | || __/ # |_____|_| |_|\__,_|_|_| \___|\___|\__| |_|\___|_| |_| |_| .__/|_|\__,_|\__\___| -# | | -# |_| +# | | +# |_| # This test tests the following scenario: # # - Create a new repository based on the PTE template with no apps (this will be the "indirect" template repository) @@ -92,7 +92,7 @@ $buildALGoProjectWorkflow = Join-Path $templateRepoPath '.github/workflows/_Buil $buildYaml = [yaml]::Load($buildALGoProjectWorkflow) $buildYaml | Should -Not -BeNullOrEmpty -# Modify the permissions +# Modify the permissions $buildYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) # Add customization steps @@ -118,7 +118,8 @@ $buildYaml.Save($buildALGoProjectWorkflow) $cicdWorkflow = Join-Path $templateRepoPath '.github/workflows/CICD.yaml' $cicdYaml = [yaml]::Load($cicdWorkflow) $cicdYaml | Should -Not -BeNullOrEmpty -# Modify the permissions + +# Modify the permissions $cicdYaml.Replace('permissions:/contents: read', @('contents: write', 'issues: read')) $customJobs = @( @{ @@ -191,7 +192,8 @@ $buildYaml.Save($buildALGoProjectWorkflow) $cicdWorkflow = Join-Path $repoPath '.github/workflows/CICD.yaml' $cicdYaml = [yaml]::Load($cicdWorkflow) $cicdYaml | Should -Not -BeNullOrEmpty -# Modify the permissions + +# Modify the permissions $cicdYaml.Replace('permissions:/contents: read', @('contents: read', 'issues: write')) $customJobs = @( From f19dda62f97766e18d8f77b4a6adff358c04f39e Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 7 Aug 2024 22:04:58 +0200 Subject: [PATCH 038/108] add pipeline init and finalize --- Actions/AL-Go-Helper.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 3c7fbdc03..d34ae33c5 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -18,7 +18,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent -$defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" +$defaultBcContainerHelperVersion = "https://github.com/freddydk/navcontainerhelper/archive/refs/heads/runpipeline.zip" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" $notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName") $runAlPipelineOverrides = @( @@ -38,6 +38,8 @@ $runAlPipelineOverrides = @( "InstallMissingDependencies" "PreCompileApp" "PostCompileApp" + "PipelineInitialize" + "PipelineFinalize" ) # Well known AppIds From a2e7b2b642e34ab2bf1f2894d07304be071c3cdb Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 8 Aug 2024 13:24:51 +0200 Subject: [PATCH 039/108] support cicdauthcontext --- Actions/AL-Go-Helper.ps1 | 2 +- Actions/RunPipeline/RunPipeline.ps1 | 15 ++++++++++++++- .../.github/workflows/_BuildALGoProject.yaml | 2 +- .../.github/workflows/_BuildALGoProject.yaml | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index d34ae33c5..1c1bc0d48 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -19,7 +19,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent $defaultBcContainerHelperVersion = "https://github.com/freddydk/navcontainerhelper/archive/refs/heads/runpipeline.zip" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" -$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName") +$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","Environment") $runAlPipelineOverrides = @( "DockerPull" diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index b102a3a91..c76d943f0 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -80,7 +80,7 @@ try { $appBuild = $settings.appBuild $appRevision = $settings.appRevision - 'licenseFileUrl','codeSignCertificateUrl','*codeSignCertificatePassword','keyVaultCertificateUrl','*keyVaultCertificatePassword','keyVaultClientId','gitHubPackagesContext','applicationInsightsConnectionString' | ForEach-Object { + 'licenseFileUrl','codeSignCertificateUrl','*codeSignCertificatePassword','keyVaultCertificateUrl','*keyVaultCertificatePassword','keyVaultClientId','gitHubPackagesContext','applicationInsightsConnectionString','cicdAuthContext' | ForEach-Object { # Secrets might not be read during Pull Request runs if ($secrets.Keys -contains $_) { $value = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$_")) @@ -186,6 +186,19 @@ try { } $authContext = $null $environmentName = "" + if ($cicdAuthContext) { + $authContext = $cicdAuthContext | ConvertFrom-Json | ConvertTo-HashTable + if ($authContext.ContainsKey('environmentName')) { + $environmentName = $authContext.environmentName + $authContext.Remove('environmentName') + if ($environmentName -notlike 'https://*') { + $authContext = New-BcAuthContext @authContext + } + } + else { + Write-Host "::WARNING::CI/CD AuthContext is missing environmentName, ignoring cicdAuthContext secret." + } + } $CreateRuntimePackages = $false if ($settings.versioningStrategy -eq -1) { diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index cafa9d978..a875964ff 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -106,7 +106,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS' + getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS,cicdAuthContext' - name: Determine ArtifactUrl uses: microsoft/AL-Go-Actions/DetermineArtifactUrl@main diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index cafa9d978..a875964ff 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -106,7 +106,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS' + getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS,cicdAuthContext' - name: Determine ArtifactUrl uses: microsoft/AL-Go-Actions/DetermineArtifactUrl@main From 91041b11b0c27f7dbfae57b341231a667bd3db02 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 8 Aug 2024 13:34:27 +0200 Subject: [PATCH 040/108] env name --- Actions/AL-Go-Helper.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 1c1bc0d48..3a02f9ef7 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -19,7 +19,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent $defaultBcContainerHelperVersion = "https://github.com/freddydk/navcontainerhelper/archive/refs/heads/runpipeline.zip" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" -$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","Environment") +$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","EnvironmentName") $runAlPipelineOverrides = @( "DockerPull" From 492cfb5ddb5897b061d3475a13f65292067fe981 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 8 Aug 2024 22:22:01 +0200 Subject: [PATCH 041/108] dump --- Actions/RunPipeline/RunPipeline.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index c76d943f0..9be100f21 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -193,6 +193,7 @@ try { $authContext.Remove('environmentName') if ($environmentName -notlike 'https://*') { $authContext = New-BcAuthContext @authContext + Write-Host $authContext.accesstoken } } else { From 699ff4d562f1dccd652dca7c43e80fa7452cc517 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 9 Aug 2024 06:23:08 +0200 Subject: [PATCH 042/108] remove dump --- Actions/RunPipeline/RunPipeline.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index 9be100f21..c76d943f0 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -193,7 +193,6 @@ try { $authContext.Remove('environmentName') if ($environmentName -notlike 'https://*') { $authContext = New-BcAuthContext @authContext - Write-Host $authContext.accesstoken } } else { From b2e47b1e7dc7f49249374d9a0255ab0356e224ef Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 9 Aug 2024 14:23:07 +0200 Subject: [PATCH 043/108] fix tests --- Tests/CustomizedYamlSnippet.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Tests/CustomizedYamlSnippet.txt b/Tests/CustomizedYamlSnippet.txt index ae1b5ba46..0ccb32cb5 100644 --- a/Tests/CustomizedYamlSnippet.txt +++ b/Tests/CustomizedYamlSnippet.txt @@ -45,7 +45,6 @@ jobs: uses: microsoft/AL-Go-Actions/WorkflowInitialize@main with: shell: powershell - eventId: "DO0091" - name: CustomStep-MyStep1 run: | @@ -60,7 +59,6 @@ jobs: uses: microsoft/AL-Go-Actions/ReadSettings@main with: shell: powershell - parentTelemetryScopeJson: ${{ steps.init.outputs.telemetryScopeJson }} - name: CustomStep-MyStep2 run: | @@ -78,14 +76,12 @@ jobs: uses: microsoft/AL-Go-Actions/ReadSettings@main with: shell: powershell - parentTelemetryScopeJson: ${{ needs.Initialization.outputs.telemetryScopeJson }} get: templateUrl - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - parentTelemetryScopeJson: ${{ needs.Initialization.outputs.telemetryScopeJson }} templateUrl: ${{ env.templateUrl }} CustomJob-MyJob: From 1cc3a947178e3c4f271500dceb18ab69f3597c7a Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 06:52:02 +0200 Subject: [PATCH 044/108] fix aldoc if --- Templates/AppSource App/.github/workflows/CICD.yaml | 2 +- Templates/Per Tenant Extension/.github/workflows/CICD.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index d57c5c899..c6f19ed66 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -166,7 +166,7 @@ jobs: DeployALDoc: needs: [ Initialization, Build ] - if: (!cancelled()) && needs.Build.result == 'Success' && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' + if: (!cancelled()) && (needs.Build.result == 'success' || needs.Build.result == 'skipped') && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' runs-on: [ windows-latest ] name: Deploy Reference Documentation permissions: diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 3d9933ee4..0dc128768 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -180,7 +180,7 @@ jobs: DeployALDoc: needs: [ Initialization, Build ] - if: (!cancelled()) && needs.Build.result == 'Success' && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' + if: (!cancelled()) && (needs.Build.result == 'success' || needs.Build.result == 'skipped') && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' runs-on: [ windows-latest ] name: Deploy Reference Documentation permissions: From b48f6753ddb1c05536234fdf461409087a831f95 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:19:38 +0200 Subject: [PATCH 045/108] Replace template owner --- Internal/Deploy.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 62eba2435..678dc1033 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -252,6 +252,12 @@ try { } Set-Content -Path (Join-Path "./.github" "RELEASENOTES.copy.md") -Value $releaseNotes -Encoding utf8 } + # Replace template_owner in README.md + $readmeFile = Join-Path $baseRepoPath "README.md" + $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" + $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") + Set-Content -Path $readmeFile -Encoding utf8 -Value $readme + # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit } From 7c986d9f5408e8a417f597477a788e1cab02f646 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:24:05 +0200 Subject: [PATCH 046/108] remove jekyll --- .github/workflows/jekyll-gh-pages.yml | 49 --------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml deleted file mode 100644 index f5916b31d..000000000 --- a/.github/workflows/jekyll-gh-pages.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Jekyll with GitHub Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@b178f9334b208360999a0a57b523613563698c66 # v1.0.12 - with: - source: . - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 - with: - preview: true From 39023cfe63e539cc6034b1dc507a6ede89ced238 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:29:40 +0200 Subject: [PATCH 047/108] modify readme --- Internal/Deploy.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 678dc1033..c7b938acf 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -116,7 +116,7 @@ try { "appSourceAppRepo" = "$($config.githubOwner)/$($config.appSourceAppRepo)" } - if ($config.branch -eq 'preview') { + if ($config.branch -eq 'preview' -or $config.githubOwner -ne 'microsoft') { # When deploying to preview, we are NOT going to deploy to a branch in the AL-Go-Actions repository # Instead, we are going to have AL-Go-PTE and AL-Go-AppSource point directly to the SHA in AL-Go $dstOwnerAndRepo += @{ @@ -252,11 +252,13 @@ try { } Set-Content -Path (Join-Path "./.github" "RELEASENOTES.copy.md") -Value $releaseNotes -Encoding utf8 } + # Replace template_owner in README.md - $readmeFile = Join-Path $baseRepoPath "README.md" + $readmeFile = './README.md' $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") Set-Content -Path $readmeFile -Encoding utf8 -Value $readme + # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit From 0c69b29608012ffb393159f584652ba810072d4e Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:33:00 +0200 Subject: [PATCH 048/108] dump readme --- Internal/Deploy.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index c7b938acf..e60f5533e 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -256,8 +256,10 @@ try { # Replace template_owner in README.md $readmeFile = './README.md' $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" + Write-Host $readme $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") Set-Content -Path $readmeFile -Encoding utf8 -Value $readme + Write-Host $readme # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit From 821aa813402d94deaa8d84d71963e1e461e30328 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:37:09 +0200 Subject: [PATCH 049/108] dump --- Internal/Deploy.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index e60f5533e..6c5f371a7 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -256,9 +256,11 @@ try { # Replace template_owner in README.md $readmeFile = './README.md' $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" + Write-Host "------------------------------------------" Write-Host $readme $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") Set-Content -Path $readmeFile -Encoding utf8 -Value $readme + Write-Host "==========================================" Write-Host $readme # Push changes From 525048b6aab1204cdd927f844212acd0aea1b04c Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 11 Aug 2024 19:39:38 +0200 Subject: [PATCH 050/108] replace --- Internal/Deploy.ps1 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 6c5f371a7..122cab4ee 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -256,12 +256,8 @@ try { # Replace template_owner in README.md $readmeFile = './README.md' $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" - Write-Host "------------------------------------------" - Write-Host $readme - $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") + $readme = $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") Set-Content -Path $readmeFile -Encoding utf8 -Value $readme - Write-Host "==========================================" - Write-Host $readme # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit From 1066788650f7e47cc3b26b25a459df8d0b794550 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 10:39:32 +0200 Subject: [PATCH 051/108] merge if permissions are present --- Actions/CheckForUpdates/yamlclass.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 7f463ed47..d58f41fd0 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,7 +445,9 @@ class Yaml { Write-host "Merge permissions" $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) - $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + if ($srcPermissions -and $yamlPermissions) { + $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + } # Apply cystom steps Write-Host "Apply custom steps" From 82585722f4245092e9b6623a82c04739819e1a8a Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 10:58:27 +0200 Subject: [PATCH 052/108] check --- Actions/CheckForUpdates/yamlclass.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index d58f41fd0..c171b6d83 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,6 +445,8 @@ class Yaml { Write-host "Merge permissions" $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) + Write-Host "'$srcPermissions'" + Write-Host "'$yamlPermissions'" if ($srcPermissions -and $yamlPermissions) { $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) } From 32e7f1bae09765c4bcc53a835a865e1e72ddc0db Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 11:07:20 +0200 Subject: [PATCH 053/108] check not '' --- Actions/CheckForUpdates/yamlclass.ps1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index c171b6d83..d8ae1016d 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -445,9 +445,7 @@ class Yaml { Write-host "Merge permissions" $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) - Write-Host "'$srcPermissions'" - Write-Host "'$yamlPermissions'" - if ($srcPermissions -and $yamlPermissions) { + if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) } From ddd95b158bca22906256f9534e46d559246d4ddc Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 11:22:23 +0200 Subject: [PATCH 054/108] add dump --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 1d84a6c40..6c84d6814 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -222,7 +222,12 @@ foreach($checkfile in $checkfiles) { [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $customizationAnchors) } + Write-Host "Checking $dstFile" $dstFileExists = Test-Path -Path $dstFile -PathType Leaf + Write-Host "DstFileExists: $dstFileExists" + Write-Host "Filename: $fileName" + Write-Host "DstPath: $dstPath" + $unusedALGoSystemFiles | Out-Host if ($unusedALGoSystemFiles -contains $fileName) { # file is not used by ALGo, remove it if it exists # do not add it to $updateFiles if it does not exist From 96818d74acc0ec40d5a8bafc2b4500d756268037 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 11:35:01 +0200 Subject: [PATCH 055/108] remove unused files not present in template --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 6c84d6814..42dcdc670 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -178,6 +178,14 @@ foreach($checkfile in $checkfiles) { UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -additionalSettings $indirectTemplateProjectSettings $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } } + + $unusedALGoSystemFiles | ForEach-Object { + if (Test-Path -Path (Join-Path $dstFolder $_) -PathType Leaf) { + Write-Host "Remove unused AL-Go system file: $_" + $removeFiles += @(Join-Path $dstPath $_) + } + } + # Loop through all files in the template repository matching the pattern Get-ChildItem -Path $srcFolder -Filter $checkfile.pattern | ForEach-Object { # Read the template file and modify it based on the settings From c97f705fb9e52a022bcdf5b3a222ecc65e9f1927 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 13 Aug 2024 11:42:43 +0200 Subject: [PATCH 056/108] move unused check --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 42 ++++++++------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 42dcdc670..9d734f475 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -179,6 +179,7 @@ foreach($checkfile in $checkfiles) { $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } } + # Remove unused AL-Go system files $unusedALGoSystemFiles | ForEach-Object { if (Test-Path -Path (Join-Path $dstFolder $_) -PathType Leaf) { Write-Host "Remove unused AL-Go system file: $_" @@ -230,36 +231,25 @@ foreach($checkfile in $checkfiles) { [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile, $customizationAnchors) } - Write-Host "Checking $dstFile" - $dstFileExists = Test-Path -Path $dstFile -PathType Leaf - Write-Host "DstFileExists: $dstFileExists" - Write-Host "Filename: $fileName" - Write-Host "DstPath: $dstPath" - $unusedALGoSystemFiles | Out-Host - if ($unusedALGoSystemFiles -contains $fileName) { - # file is not used by ALGo, remove it if it exists - # do not add it to $updateFiles if it does not exist - if ($dstFileExists) { - $removeFiles += @(Join-Path $dstPath $filename) + if ($unusedALGoSystemFiles -notcontains $fileName) { + if (Test-Path -Path $dstFile -PathType Leaf) { + if ($type -eq 'workflow') { + Write-Host "Apply customizations from my repository: $dstFile" + [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $customizationAnchors) + } + # file exists, compare and add to $updateFiles if different + $dstContent = Get-ContentLF -Path $dstFile + if ($dstContent -cne $srcContent) { + Write-Host "Updated $type ($(Join-Path $dstPath $filename)) available" + $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } + } } - } - elseif ($dstFileExists) { - if ($type -eq 'workflow') { - Write-Host "Apply customizations from my repository: $dstFile" - [Yaml]::ApplyCustomizations([ref] $srcContent,$dstFile, $customizationAnchors) - } - # file exists, compare and add to $updateFiles if different - $dstContent = Get-ContentLF -Path $dstFile - if ($dstContent -cne $srcContent) { - Write-Host "Updated $type ($(Join-Path $dstPath $filename)) available" + else { + # new file, add to $updateFiles + Write-Host "New $type ($(Join-Path $dstPath $filename)) available" $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } } } - else { - # new file, add to $updateFiles - Write-Host "New $type ($(Join-Path $dstPath $filename)) available" - $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } - } } } finally { From 01b6d49b904f0b54eacbfd5d0d9759398bcdb17d Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 18 Aug 2024 21:32:50 +0200 Subject: [PATCH 057/108] add customALGoSystemFiles --- Actions/AL-Go-Helper.ps1 | 1 + .../CheckForUpdates.HelperFunctions.ps1 | 96 ++++++ Actions/CheckForUpdates/CheckForUpdates.ps1 | 3 + README.md | 1 + RELEASENOTES.md | 7 + Scenarios/Contribute.md | 17 +- Scenarios/CustomizingALGoForGitHub.md | 304 ++++++++++++++++++ Scenarios/settings.md | 10 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 4 +- 9 files changed, 429 insertions(+), 14 deletions(-) create mode 100644 Scenarios/CustomizingALGoForGitHub.md diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 3a02f9ef7..73c731ad9 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -554,6 +554,7 @@ function ReadSettings { $settings = [ordered]@{ "type" = "PTE" "unusedALGoSystemFiles" = @() + "customALGoSystemFiles" = @() "projects" = @() "powerPlatformSolutionFolder" = "" "country" = "us" diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 466993bea..ead9cc0fb 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -463,3 +463,99 @@ function UpdateSettingsFile { # Save the file with LF line endings and UTF8 encoding $settings | Set-JsonContentLF -path $settingsFile } + +function ApplyCustomALGoSystemFiles { + Param( + [string] $path, + [hashtable] $settings + ) + + Push-Location -Path $path + try { + if ($settings.customALGoSystemFiles -isnot [Array]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + foreach($customspec in $settings.customALGoSystemFiles) { + if ($customspec -isnot [Hashtable]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (!($customSpec.ContainsKey('Source') -and $customSpec.ContainsKey('Destination'))) { + throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be specified. See https://aka.ms/algosettings#customalgosystemfiles." + } + $source = $customspec.Source + $destination = $customSpec.Destination + if ($source -isnot [string] -or $destination -isnot [string]) { + throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be strings. See https://aka.ms/algosettings#customalgosystemfiles." + } + + $finalDestination = Join-Path $path $destination + $ext = [System.IO.Path]::GetExtension($source) + if ($ext -eq '.zip') { + Write-Host $destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) + if ($customSpec.ContainsKey('FileSpec')) { $fileSpec = $customSpec.FileSpec } else { $fileSpec = '*' } + if ($customSpec.ContainsKey('Recurse')) { $recurse = $customSpec.Recurse } else { $recurse = $true } + if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { + throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (Test-Path $finalDestination -PathType Leaf) { + throw "A file with that name already exists - destination must point to a folder when adding custom AL-Go System Files from a .zip file. See https://aka.ms/algosettings#customalgosystemfiles." + } + $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + New-Item -Path $tempFolder -ItemType Directory | Out-Null + $zipName = "$tempFolder$ext" + try { + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName + Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force + $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve + Push-Location -Path $subFolder + try { + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + $destRelativeFileName = Resolve-Path $_.FullName -Relative + $destFinalFileName = Join-Path $finalDestination $destRelativeFileName + $destFinalFolder = [System.IO.Path]::GetDirectoryName($destFinalFileName) + if (Test-Path -Path $destFinalFolder -PathType Leaf) { + throw "A file ($destFinalFolder) already exists, where a folder is expected when applying CustomALGoSystemFiles." + } + if (!(Test-Path -Path $destFinalFolder)) { + New-Item -path $destFinalFolder -ItemType Directory | Out-Null + } + Write-Host "- $($destRelativeFileName.Substring(2))" + Copy-Item -Path $_.FullName -Destination $destFinalFileName + } + } + finally { + Pop-Location + } + } + finally { + if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } + Remove-Item -Path $tempFolder -Recurse -Force + } + } + else { + if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { + throw "customALGoSystemFiles setting is wrongly formatted, FileSpec and Recurse are only allowed with .zip files. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (Test-Path -Path $finalDestination -PathType Container) { + $finalDestination = Join-Path $finalDestination ([System.IO.Path]::GetFileName($source)) + } + else { + if ($destination.endsWith('\') -or $destination.endsWith('/')) { + New-Item -Path $finalDestination -ItemType Directory | Out-Null + $finalDestination = Join-Path $finalDestination ([System.IO.Path]::GetFileName($source)) + } + } + $overridingStr = '' + if (Test-Path -Path $finalDestination -PathType Leaf) { + $overridingStr = ' (Overriding existing file)' + } + $relativeFolder = Resolve-Path -Path ([System.IO.Path]::GetDirectoryName($finalDestination)) -Relative + Write-Host "$(Join-Path $relativeFolder ([System.IO.Path]::GetFileName($finalDestination)))$overridingStr" + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $finalDestination + } + } + } + finally { + Pop-Location + } +} \ No newline at end of file diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 9d734f475..ff4c367c6 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -116,6 +116,9 @@ if (-not $isDirectALGo) { } } +# Apply Custom AL-Go System Files from settings +ApplyCustomALGoSystemFiles -path $templateFolder -settings $repoSettings + # CheckFiles is an array of hashtables with the following properties: # dstPath: The path to the file in the current repository # srcPath: The path to the file in the template repository diff --git a/README.md b/README.md index 2ff4b733e..d67e0bf67 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Try out the [AL-Go workshop](https://aka.ms/algoworkshop) for an in-depth worksh 1. [Connect your GitHub repository to Power Platform](Scenarios/SetupPowerPlatform.md) 1. [How to set up Service Principal for Power Platform](Scenarios/SetupServicePrincipalForPowerPlatform.md) 1. [Try one of the Business Central and Power Platform samples](Scenarios/TryPowerPlatformSamples.md) +1. [Customizing AL-Go for GitHub](Scenarios/CustomizingALGoForGitHub.md) ## Migration scenarios diff --git a/RELEASENOTES.md b/RELEASENOTES.md index c88af836a..9abaac27e 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -47,6 +47,13 @@ AL-Go for GitHub now includes a new telemetry module. For detailed information o - **NumberOfSqlStmtsWarning** - a warning is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 5) - **NumberOfSqlStmtsError** - an error is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 10) +- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: + + - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) + - **Source** (mandatory) - URL to a either a single file or a .zip file containing + - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. + - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) + > \[!NOTE\] > Duration thresholds are subject to varying results depending on the performance of the agent running the tests. Number of SQL statements executed by a test is often the most reliable indicator of performance degredation. diff --git a/Scenarios/Contribute.md b/Scenarios/Contribute.md index 8ff23d670..0bca431d8 100644 --- a/Scenarios/Contribute.md +++ b/Scenarios/Contribute.md @@ -5,16 +5,16 @@ This section describes how to contribute to AL-Go. How to set up your own enviro You can do this in two ways: - Use a fork of AL-Go for GitHub in your own **personal GitHub account** in development mode -- Use 3 public repositories in your own **personal GitHub account** (AL-Go-PTE, AL-Go-AppSource and AL-Go-Actions, much like in production) +- Use 2 public repositories in your own **personal GitHub account** (AL-Go-PTE, AL-Go-AppSource and AL-Go-Actions, much like in production) ## Use a fork of AL-Go for GitHub in "development mode" 1. Fork the [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go) repository to your **personal GitHub account**. 1. You can optionally also create a branch in the AL-Go fork for the feature you are working on. -**https://github.com//AL-Go@** can now be used as a template in your AL-Go project when running _Update AL-Go System Files_ to use the actions/workflows from this fork. +**/AL-Go@** can now be used as a template in your AL-Go project when running _Update AL-Go System Files_ to use the actions/workflows from this fork. -## Use 3 public repositories in "production mode" +## Use 2 public repositories in "production mode" 1. Fork the [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go) repository to your **personal GitHub account**. 1. Navigate to [https://github.com/settings/tokens/new](https://github.com/settings/tokens/new) and create a new personal access token with **Full control of private repositories** and **workflow** permissions. @@ -22,15 +22,11 @@ You can do this in two ways: 1. In your personal fork of AL-Go, navigate to **Actions**, select the **Deploy** workflow and choose **Run Workflow**. 1. Using the default settings press **Run workflow**. Select the AL-Go branch to run from and the branch to deploy to. -Now you should have 3 new public repositories: +Now you should have 2 new public repositories: -- [https://github.com/yourGitHubUserName/AL-Go-Actions](https://github.com/yourGitHubUserName/AL-Go-Actions) - [https://github.com/yourGitHubUserName/AL-Go-AppSource](https://github.com/yourGitHubUserName/AL-Go-AppSource) - [https://github.com/yourGitHubUserName/AL-Go-PTE](https://github.com/yourGitHubUserName/AL-Go-PTE) -> \[!NOTE\] -> Deploying to a branch called **preview** will only update the two template repositories (and use your AL-Go project with the current SHA as actions repository). - You can optionally also create a branch in the AL-Go fork for the feature you are working on and then select that branch when running Deploy (both as **Use workflow from** and as **Branch to deploy to**). **yourGitHubUserName/AL-Go-PTE@yourBranch** or **yourGitHubUserName/AL-Go-AppSource@yourBranch** can now be used in your AL project when running Update AL-Go System Files to use the actions/workflows from this area for your AL project. @@ -38,7 +34,7 @@ You can optionally also create a branch in the AL-Go fork for the feature you ar Please ensure that all unit tests run and create a Pull Request against [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go). You are very welcome to run the end to end tests as well, but we will also run the end to end tests as part of the code review process. > \[!NOTE\] -> You can also deploy to a different branch in the 3 public repositories by specifying a branch name under **Branch to deploy** to when running the **Deploy** workflow. The branch you specify in **Use workflow from** indicates which branch in **your personal fork of the AL-Go repository** you publish to the 3 repositories. +> You can also deploy to a different branch in the 2 public repositories by specifying a branch name under **Branch to deploy** to when running the **Deploy** workflow. The branch you specify in **Use workflow from** indicates which branch in **your personal fork of the AL-Go repository** you publish to the 2 repositories. ## Pre-Commit @@ -57,9 +53,8 @@ In the e2eTests folder, in the AL-Go repository, there are 3 types of end to end - The scenarios folder contains a set of AL-Go scenarios, which tests specific functionality end to end. Every folder under e2eTests/scenarios, which contains a runtests.ps1 will be run as a scenario test, like: - UseProjectDependencies - create a repo with multiple projects and set **UseProjectDependencies** to modify CI/CD and other build workflows to build projects in the right order - GitHubPackages - create 3 repositories using GitHub Packages as dependency resolver and check that artifacts are built properly - - BuildModes - create a repository, set buildModes and test that generated artifacts are as expected. - - ReleaseBranches - testing that create release works, release branches are create and subsequently found correctly as previous build - SpecialCharacters - testing that various settings (+ publisher name and app name) can contain special national characters + - and more... In your personal fork, you can now run the end to end tests, if the following pre-requisites are available: diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md new file mode 100644 index 000000000..3867f89be --- /dev/null +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -0,0 +1,304 @@ +# Customizing AL-Go for GitHub + +AL-Go for GitHub is a plug-and-play DevOps solution, intended to support 100% of the functionality needed by 90% of the people developing applications for Microsoft Dynamics 365 Business Central out-of-the-box. + +If AL-Go functionality out-of-the-box doesn't match your needs, you really three options: + +1. Customize AL-Go for GitHub to fit your needs +1. Select another managed DevOps solution +1. Create your own DevOps solution from scratch (not recommended) + +Creating your own DevOps solution from scratch requires dedicated resources to develop and maintain workflows, processes etc. **This is not a small task**. There are many moving parts in a DevOps solution, which might require you to make changes to workflows and scripts over time and stay secure and having to maintain many repositories is tedious and time consuming, even when using templates and other advanced features. + +Microsoft will continuously develop and maintain AL-Go for GitHub and ensure that we always use the latest versions of GitHub actions, which are under our control. Microsoft will never add dependencies to any third party GitHub action, which are not under our control. + +Keeping your repositories up-to-date can be done manually or on a schedule (like Windows update really). You will be notified when an update is available and we recommend that you keep your repositories up-to-date at all time. If you make modifications to the AL-Go System Files (scripts and workflows) in your repository, in other ways than described in this document, these changes will be removed with the next AL-Go update. + +> \[!TIP] +> If for some reason the updated version of AL-Go for GitHub doesn't work for you, we recommend that you file an issue [here](https://github.com/microsoft/AL-Go/issues) with a detailed description of the problem and full logs of the failing workflows. You can then revert back to the prior version of AL-Go for GitHub until the issue is resolved. +> +> It is important to get back to the mainstream version of AL-Go for GitHub as soon as the issue is resolved. + +There are three ways you can customize AL-Go for GitHub to fit your needs. You can + +1. customize the repository with custom scripts, workflows, jobs or steps following the guidelines below +1. create a customized repository and use this as your template repository (indirect template) +1. fork the AL-Go for GitHub and create your "own" version + +> \[!CAUTION\] +> The more you customize AL-Go for GitHub, the more likely you are to be broken by future updates to AL-Go for GitHub, meaning that you will have to update your customizations to match the changes in AL-Go for GitHub. + +## Customizing your repository + +There are several ways you can customize your AL-Go repository and ensure that the changes you make, will survive an update of AL-Go for GitHub. + +### Hide/Remove unused workflows + +By adding a setting called [`unusedALGoSystemFiles`](https://aka.ms/algosettings#unusedalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these system files are not used. Example: + +``` + "unusedALGoSystemFiles": [ + "AddExistingAppOrTestApp.yaml", + "CreateApp.yaml", + "CreatePerformanceTestApp.yaml", + "CreateTestApp.yaml", + "cloudDevEnv.ps1" + ] +``` + +This setting will cause AL-Go for GitHub to remove these files during the next update. Note that if you remove files like `_BuildALGoProject.yaml`, AL-Go will obviously stop working as intended - so please use with care. + +### Custom delivery + +You can setup [custom delivery](https://aka.ms/algosettings#customdelivery) in order to deliver your apps to locations not supported by AL-Go for GitHub out-of-the-box, by adding a custom delivery powershell script (named `.github/DeliverTo.ps1`) and a context secret (called `Context`) formatted as compressed json, you can define the delivery functionality as you like. Example: + +```powershell +Param([Hashtable] $parameters) + +Get-ChildItem -Path $parameters.appsFolder | Out-Host +$context = $parameters.context | ConvertFrom-Json +Write-Host "Token Length: $($context.Token.Length)" +``` + +In this example the context secret is assumed to contain a Token property. Read [this](https://aka.ms/algosettings#customdelivery) for more information. + +### Custom deployment + +You can setup [custom deployment](https://aka.ms/algosettings#customdeployment) to environment types not supported by AL-Go for GitHub out-of-the-box. You can also override deployment functionality to environment Type `SaaS` if you like. You can add an environment called `` and a `DeployTo` setting, defining which environment Type should be used. Example: + +```json + "Environments": [ + "" + ], + "DeployTo": { + "EnvironmentType": "" + } +``` + +You also need to create an AuthContext secret (called `_AuthContext`) and a powershell script (named `.github/DeployTo.ps1`), which defines the deployment functionality. Example: + +```powershell +Param([Hashtable] $parameters) + +$parameters | ConvertTo-Json -Depth 99 | Out-Host +$tempPath = Join-Path ([System.IO.Path]::GetTempPath()) ([GUID]::NewGuid().ToString()) +New-Item -ItemType Directory -Path $tempPath | Out-Null +Copy-AppFilesToFolder -appFiles $parameters.apps -folder $tempPath | Out-Null +Get-ChildItem -Path $tempPath -Filter *.app | Out-Host +$authContext = $parameters.authContext | ConvertFrom-Json +Write-Host "Token Length: $($authContext.Token.Length)" +``` + +In this example the AuthContext secret is assumed to contain a Token property. Read [this](https://aka.ms/algosettings#customdeployment) for more information. + +### Adding custom workflows + +If you add new workflows to the `.github/workflows` folder, which is unknown to AL-Go for GitHub, AL-Go will leave them un-touched. These workflows needs to follow standard GitHub Actions schema (yaml) and can be triggered as any other workflows. Example: + +```yaml +name: 'Create Build Tag' + +on: + workflow_run: + workflows: [' CI/CD','CI/CD'] + types: [completed] + branches: [ 'main' ] + +run-name: "[${{ github.ref_name }}] Create build tag" + +permissions: read-all + +jobs: + CreateTag: + if: github.event.workflow_run.conclusion == 'success' + runs-on: windows-latest + steps: + - name: mystep + run: | + Write-Host "Create tag" +``` + +It is recommended to prefix your workflows with `my`, `our`, your name or your organization name in order to avoid that the workflow suddenly gets overridden by a new workflow in AL-Go for GitHub. The above workflow is a real example from [here](https://github.com/microsoft/BCApps/blob/main/.github/workflows/CreateBuildTag.yaml). + +> \[!CAUTION\] +> This workflow gets triggered when the CI/CD workflow has completed. Note that the name of the CI/CD workflow currently is prefixed with a space, this space will very likely be removed in the future, which is why we specify both names in this example. Obviously this workflow would break if we decide to rename the CI/CD workflow to something different. + +### Adding custom scripts + +You can add custom powershell scripts under the .github folder for repository scoped scripts or in the .AL-Go folder for project scoped scripts. Specially named scripts in the .AL-Go folder can override standard functionality in AL-Go for GitHub workflows. A list of these script overrides can be found [here](https://aka.ms/algosettings#customdeployment). Scripts under the .github folder can be used in custom workflows instead of using inline scripts inside the workflow. + +One example of a script override is the NewBcContainer override used in the System Application project in BCApps (can be found [here](https://github.com/microsoft/BCApps/blob/main/build/projects/System%20Application/.AL-Go/NewBcContainer.ps1)). This override looks like: + +```powershell +Param([Hashtable] $parameters) + +$script = Join-Path $PSScriptRoot "../../../scripts/NewBcContainer.ps1" -Resolve +. $script -parameters $parameters +``` + +Which basically launches a script located in the script folder in the repository for creating the build container needed for building and testing the System Application. + +> \[!CAUTION\] +> Script overrides will almost certainly be broken in the future. The current script overrides is very much tied to the current implementation of the `Run-AlPipeline` function in BcContainerHelper. In the future, we will move this functionality to GitHub actions and no longer depend on BcContainerHelper and Run-AlPipeline. At that time, these script overrides will have to be changed to follow the new implementation. + + + +### Adding custom workflows and/or scripts using a URL + +By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings#customalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these files should be included in the update. Example: + +``` + "customALGoSystemFiles": [ + { + "Destionation": ".AL-Go/", + "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1", + }, + { + "Destionation": ".AL-Go/", + "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1", + }, + { + "Destionation": ".AL-Go/NewBcContainer.ps1", + "Source": "https://raw.githubusercontent.com/microsoft/BCApps/main/build/scripts/NewBcContainer.ps1", + }, + { + "Destination": ".github/", + "Source": "https://github.com/freddydk/CustomALGoSystemFiles/archive/refs/heads/main.zip", + "FileSpec": "*/.github/*", + "Recurse": true + } + ] +``` + +`customALGoSystemFiles` is an array of objects, which currently can have 4 properties: + +| Property | Description | Mandatory | Default | +| :-- | :-- | :-: | :-- | +| Destination | Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) | Yes | | +| Source | URL to a either a single file or a .zip file containing | Yes | | +| FileSpec | If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | +| Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | + +This setting will cause AL-Go for GitHub to include these files during the next update. + +> \[!WARNING\] +> You can override existing AL-Go for GitHub system files this way, please prefix files in your repository with `my` or your organization name (except for DeployTo and DeliverTo) in order to avoid overriding future workflows from AL-Go for GitHub. + +> \[!NOTE\] +> If the destination is in the .AL-Go folder, the file(s) will be copied to all .AL-Go folders in multi-project repositories. + +### Adding custom jobs + +You can also add custom jobs to any of the existing AL-Go for GitHub workflows. Custom jobs can depend on other jobs and other jobs can made to depend on custom jobs. Custom jobs needs to be named `CustomJob`, but can specify another name to be shown in the UI. Example: + +```yaml + CustomJob-CreateBuildTag: + name: Create Build Tag + needs: [ Initialization, Build ] + if: (!cancelled()) && (needs.Build.result == 'success') + runs-on: [ ubuntu-latest ] + steps: + - name: Create Tag + run: | + Write-Host "Create Tag" + + PostProcess: + needs: [ Initialization, Build2, Build1, Build, Deploy, Deliver, DeployALDoc, CustomJob-CreateBuildTag ] + if: (!cancelled()) + runs-on: [ windows-latest ] + steps: + ... +``` + +Adding a custom job like this, will cause this job to run simultaneously with the deploy and the deliver jobs. + +> \[!NOTE\] +> All custom jobs will be moved to the tail of the yaml file when running Update AL-Go System Files, but dependencies to/from the custom jobs will be maintained. + +> \[!CAUTION\] +> Custom jobs might be broken if the customized AL-Go for GitHub workflow has been refactored and the referenced jobs have been renamed. + +### Adding custom steps + +You can also add custom steps to AL-Go for GitHub Workflows, but only in pre-defined anchor-points. The reason for only allowing custom steps at pre-defined anchor-points is that we want to limit the number of places where steps can be added in order to have some level of freedom to refactor, develop and maintain the AL-Go for GitHub workflows, without breaking customizations constantly. + +At this time, the anchor-points where you can add custom steps are: + +| Workflow | Job | Step | Before or AFter | +| :-- | :-- | :-- | :-: | +| _BuildALGoProject.yaml | BuildALGoProject | Read settings | After | +| | | Read secrets | After | +| | | Build | Before | +| | | Read secrets | After | +| | | Cleanup | Before | + +The custom step needs to be named `CustomStep` and if inserted in any of the specified anchor-points, it will be maintained after running Update AL-Go System Files. An example of a custom step could be a step, which modifies settings based on some business logic + +```yaml + - name: CustomStep-ModifySettings + run: | + $settings = $env:Settings | ConvertFrom-Json + $settings.artifact = Invoke-RestMethod -Method GET -UseBasicParsing -Uri "https://bca-url-proxy.azurewebsites.net/bca-url/sandbox/us?select=weekly&doNotRedirect=true" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "Settings=$($$settings | ConvertTo-Json -Depth 99 -Compress)" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "artifact=$($settings.artifact)" +``` + +> \[!TIP\] +> Create a feature request [here](https://github.com/microsoft/AL-Go/issues/new?assignees=&labels=enhancement&projects=&template=enhancement.yaml&title=%5BEnhancement%5D%3A+) with a description on where you would like additional anchor-points and what you want to use it for. + +> \[!CAUTION\] +> Please be aware that changes to AL-Go for GitHub might break with future versions of AL-Go for GitHub. We will of course try to keep these breaking changes to a minimum, but the only way you can be sure to NOT be broken is by NOT customizing AL-Go for GitHub. + +### Modifying workflow permissions + +If any of your custom jobs require permissions, which exceeds the permissions already assigned in the workflow, then these permissions can be specified directly on the custom job. + +If any of your custom steps require permissions, which exceeds the permissions already assigned in the workflow, you can modify the permissions of the workflow and assign additional permissions. AL-Go for GitHub will not allow you to remove permissions, which might be needed in other steps/jobs, but additional permissions will be included when running Update AL-Go System Files. + +## Using indirect templates + +If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. + +> \[!NOTE\] +> Indirect templates can be public or private. + + + +> \[!TIP\] +> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specied. + + + + +> \[!NOTE\] +> If you use the indirect template as a GitHub template, for creating the repository, then you need to re-specify the indirect Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the indirect template as it's template repository. + + + + + +## Forking AL-Go for GitHub and making your "own" **public** version + +Using a fork of AL-Go for GitHub to have your "own" public version of AL-Go for GitHub gives you the maximum customization capabilities. It does however also come with the most work. + +> \[!NOTE\] +> When customizing AL-Go for GitHub using a fork, your customizations are public and will be visible to everyone. For more information, [read this](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-permissions-and-visibility-of-forks). + +There are two ways of forking AL-Go for GitHub. You can fork the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository or you can fork the template repositories [AL-Go PTE](https://github.com/microsoft/AL-Go-PTE) and/or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource). + +For simple changes to the templates, you can fork the template repositories and make the changes directly in your fork. Note that we do not accept any pull requests to the template repositories as they are deployed from the main AL-Go repository. We do not actually develop anything in the template repositories ourself. In the template repositories you will find a branch for every version of AL-Go we have shipped. The main branch is the latest version and the preview branch is the next version. You can customize the preview branch and/or the main branch and then use your fork as the template repository when running Update AL-Go System Files from your app repositories. + +> \[!NOTE\] +> We do NOT accept pull requests to the template repositories. You need to follow the guidelines [here](Contribute.md) in order to contribute to AL-Go development. + +> \[!TIP\] +> When forking the template repositories, you should include all branches in order to be able to use either the latest version of AL-Go or the preview version of AL-Go. + +When forking the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository, you are basically developing AL-Go in the same way as we are doing in Microsoft. Please follow the guidelines [here](Contribute.md) on how to develop. This gives you maximum customization capabilities, but if your changes are not being contributed to AL-Go, then you will have to merge our changes all the time. + +> \[!CAUTION\] +> We strongly suggest that you keep your changes to a minimum and that you keep your fork up-to-date with the latest changes of AL-Go for GitHub at all time. + +______________________________________________________________________ + +[back](../README.md) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 67927c0be..6deef695e 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -4,6 +4,8 @@ The behavior of AL-Go for GitHub is very much controlled by settings and secrets To learn more about the secrets used by AL-Go for GitHub, please navigate to [Secrets](secrets.md). + + ## Where are the settings located Settings can be defined in GitHub variables or in various settings file. An AL-Go repository can consist of a single project (with multiple apps) or multiple projects (each with multiple apps). Settings can be applied on the project level or on the repository level. Multiple projects in a single repository are comparable to multiple repositories; they are built, deployed, and tested separately. All apps in each project (single or multiple) are built together in the same pipeline, published and tested together. If a repository is multiple projects, each project is stored in a separate folder in the root of the repository. @@ -210,12 +212,13 @@ Please read the release notes carefully when installing new versions of AL-Go fo | memoryLimit | Specifies the memory limit for the build container. By default, this is left to BcContainerHelper to handle and will currently be set to 8G | 8G | | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | \[ \] | +| customALGoSystemFiles | An array of objects, specifying custom AL-Go System Files. Each object must contain **Destination** and **Source** url and if the source url points to a .zip file, you can also add **FileSpec** and **Recurse**. See more [here](./CustomizingALGoForGitHub.md#customALGoSystemFiles) | \[ \] | ## Custom Delivery -You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named DeliverTo\*.ps1 in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. +You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named `DeliverTo.ps1` in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. Beside the script, you also need to create a secret called `Context`, formatted as compressed json, containing delivery information for your delivery target. ### DeliverToSharePoint.ps1 @@ -227,6 +230,7 @@ Param( Write-Host "Current project path: $($parameters.project)" Write-Host "Current project name: $($parameters.projectName)" Write-Host "Delivery Type (CD or Release): $($parameters.type)" +Write-Host "Delivery Context: $($parameters.context)" Write-Host "Folder containing apps: $($parameters.appsFolder)" Write-Host "Folder containing test apps: $($parameters.testAppsFolder)" Write-Host "Folder containing dependencies (requires generateDependencyArtifact set to true): $($parameters.dependenciesFolder)" @@ -306,6 +310,8 @@ Here are the parameters to use in your custom script: | `$parameters."runs-on"` | GitHub runner to be used to run the deployment script | windows-latest | | `$parameters."shell"` | Shell used to run the deployment script, pwsh or powershell | powershell | + + ## Run-AlPipeline script override AL-Go for GitHub utilizes the Run-AlPipeline function from BcContainerHelper to perform the actual build (compile, publish, test etc). The Run-AlPipeline function supports overriding functions for creating containers, compiling apps and a lot of other things. @@ -316,6 +322,7 @@ Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the fut | Override | Description | | :-- | :-- | +| PipelineInitialize.ps1 | Initialize the pipeline | | DockerPull.ps1 | Pull the image specified by the parameter $imageName | | NewBcContainer.ps1 | Create the container using the parameters transferred in the $parameters hashtable | | ImportTestToolkitToBcContainer.ps1 | Import the test toolkit apps specified by the $parameters hashtable | @@ -330,6 +337,7 @@ Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the fut | GetBcContainerAppRuntimePackage.ps1 | Get the runtime package specified by the $parameters hashtable | | RemoveBcContainer.ps1 | Cleanup based on the $parameters hashtable | | InstallMissingDependencies | Install missing dependencies | +| PipelineFinalize.ps1 | Finalize the pipeline | ## BcContainerHelper settings diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 9ded4de35..bfdf73edd 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -65,10 +65,10 @@ jobs: - name: Override templateUrl env: - templateUrl: ${{ github.event.inputs.templateUrl }} + newTemplateUrl: ${{ github.event.inputs.templateUrl }} run: | $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 - $templateUrl = $ENV:templateUrl + $templateUrl = $ENV:newTemplateUrl if ($templateUrl) { Write-Host "Using Template Url: $templateUrl" Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "templateUrl=$templateUrl" From c43ea9ac87752bbd1e47814e4f22da55255c143b Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 06:47:49 +0200 Subject: [PATCH 058/108] Apply custom files --- .../CheckForUpdates.HelperFunctions.ps1 | 141 ++++++++++-------- Actions/CheckForUpdates/CheckForUpdates.ps1 | 6 +- Scenarios/CustomizingALGoForGitHub.md | 12 +- 3 files changed, 84 insertions(+), 75 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index ead9cc0fb..ca19d166c 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -467,95 +467,108 @@ function UpdateSettingsFile { function ApplyCustomALGoSystemFiles { Param( [string] $path, - [hashtable] $settings + [hashtable] $settings, + [string[]] $projects ) - Push-Location -Path $path - try { - if ($settings.customALGoSystemFiles -isnot [Array]) { - throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + function CopyItem{ + Param( + [string] $source, + [string] $basePath, + [string] $destination, + [string[]] $projects + ) + + if ($destination -like ".AL-Go$([IO.Path]::DirectorySeparatorChar)*") { + $destinations = $projects | ForEach-Object { Join-Path $_ $destination } + } + else { + $destinations = @($destination) } - foreach($customspec in $settings.customALGoSystemFiles) { - if ($customspec -isnot [Hashtable]) { - throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + $destinations | ForEach-Object { + $finalDestination = Join-Path $basePath $_ + $finalDestinationFolder = [System.IO.Path]::GetDirectoryName($finalDestination) + if (!(Test-Path -Path $finalDestinationFolder -PathType Container)) { + New-Item -path $finalDestinationFolder -ItemType Directory | Out-Null } - if (!($customSpec.ContainsKey('Source') -and $customSpec.ContainsKey('Destination'))) { - throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be specified. See https://aka.ms/algosettings#customalgosystemfiles." + if (Test-Path $finalDestination) { + Write-Host "- $_ (overriding existing file)" } - $source = $customspec.Source - $destination = $customSpec.Destination - if ($source -isnot [string] -or $destination -isnot [string]) { - throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be strings. See https://aka.ms/algosettings#customalgosystemfiles." + else { + Write-Host "- $_" } - - $finalDestination = Join-Path $path $destination - $ext = [System.IO.Path]::GetExtension($source) + Copy-Item -Path $source -Destination $finalDestination + } + } + + if ($settings.customALGoSystemFiles -isnot [Array]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + foreach($customspec in $settings.customALGoSystemFiles) { + if ($customspec -isnot [Hashtable]) { + throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." + } + if (!($customSpec.ContainsKey('Source') -and $customSpec.ContainsKey('Destination'))) { + throw "customALGoSystemFiles setting is wrongly formatted, Source and Destination must be specified. See https://aka.ms/algosettings#customalgosystemfiles." + } + $source = $customspec.Source + $destination = $customSpec.Destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) + if ($destination -isnot [string] -or $destination -eq '') { + throw "customALGoSystemFiles setting is wrongly formatted, Destination must be a string, which isn't blank. See https://aka.ms/algosettings#customalgosystemfiles." + } + if ($source -isnot [string] -or $source -notlike 'https://*' -or (-not [System.Uri]::IsWellFormedUriString($source,1))) { + throw "customALGoSystemFiles setting is wrongly formatted, Source must secure download URL. See https://aka.ms/algosettings#customalgosystemfiles." + } + + $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + New-Item -Path $tempFolder -ItemType Directory | Out-Null + $ext = [System.IO.Path]::GetExtension($source) + $zipName = "$tempFolder$ext" + Push-Location -Path $path + try { if ($ext -eq '.zip') { - Write-Host $destination.Replace('/',[IO.Path]::DirectorySeparatorChar).Replace('\',[IO.Path]::DirectorySeparatorChar) + Write-Host "$($destination):" if ($customSpec.ContainsKey('FileSpec')) { $fileSpec = $customSpec.FileSpec } else { $fileSpec = '*' } if ($customSpec.ContainsKey('Recurse')) { $recurse = $customSpec.Recurse } else { $recurse = $true } if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." } - if (Test-Path $finalDestination -PathType Leaf) { - throw "A file with that name already exists - destination must point to a folder when adding custom AL-Go System Files from a .zip file. See https://aka.ms/algosettings#customalgosystemfiles." + if (!($destination.EndsWith([IO.Path]::DirectorySeparatorChar))) { + throw "customALGoSystemFiles setting is wrongly formatted, destination must be a folder (terminated with / or \). See https://aka.ms/algosettings#customalgosystemfiles." } - $tempFolder = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - New-Item -Path $tempFolder -ItemType Directory | Out-Null - $zipName = "$tempFolder$ext" + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName + Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force + $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve + Push-Location -Path $subFolder try { - Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName - Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force - $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve - Push-Location -Path $subFolder - try { - Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { - $destRelativeFileName = Resolve-Path $_.FullName -Relative - $destFinalFileName = Join-Path $finalDestination $destRelativeFileName - $destFinalFolder = [System.IO.Path]::GetDirectoryName($destFinalFileName) - if (Test-Path -Path $destFinalFolder -PathType Leaf) { - throw "A file ($destFinalFolder) already exists, where a folder is expected when applying CustomALGoSystemFiles." - } - if (!(Test-Path -Path $destFinalFolder)) { - New-Item -path $destFinalFolder -ItemType Directory | Out-Null - } - Write-Host "- $($destRelativeFileName.Substring(2))" - Copy-Item -Path $_.FullName -Destination $destFinalFileName - } - } - finally { - Pop-Location + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + $destRelativeFileName = Resolve-Path $_.FullName -Relative + $destFileName = Join-Path $destination $destRelativeFileName + $destFileName = $destFileName.TrimStart('\/') + CopyItem -source $_.FullName -basePath $path -destination $destFileName -projects $projects } } finally { - if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } - Remove-Item -Path $tempFolder -Recurse -Force + Pop-Location } } else { if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { throw "customALGoSystemFiles setting is wrongly formatted, FileSpec and Recurse are only allowed with .zip files. See https://aka.ms/algosettings#customalgosystemfiles." } - if (Test-Path -Path $finalDestination -PathType Container) { - $finalDestination = Join-Path $finalDestination ([System.IO.Path]::GetFileName($source)) + if ($destination.endsWith([IO.Path]::DirectorySeparatorChar)) { + $destination = Join-Path $destination ([System.IO.Path]::GetFileName($source)) } - else { - if ($destination.endsWith('\') -or $destination.endsWith('/')) { - New-Item -Path $finalDestination -ItemType Directory | Out-Null - $finalDestination = Join-Path $finalDestination ([System.IO.Path]::GetFileName($source)) - } - } - $overridingStr = '' - if (Test-Path -Path $finalDestination -PathType Leaf) { - $overridingStr = ' (Overriding existing file)' - } - $relativeFolder = Resolve-Path -Path ([System.IO.Path]::GetDirectoryName($finalDestination)) -Relative - Write-Host "$(Join-Path $relativeFolder ([System.IO.Path]::GetFileName($finalDestination)))$overridingStr" - Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $finalDestination + Write-Host "$($destination):" + $tempFilename = Join-Path $tempFolder ([System.IO.Path]::GetFileName($source)) + Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $tempFilename + CopyItem -source $tempFilename -basePath $path -destination $destination -projects $projects } } - } - finally { - Pop-Location + finally { + Pop-Location + if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } + Remove-Item -Path $tempFolder -Recurse -Force + } } } \ No newline at end of file diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index ff4c367c6..cc9928ac4 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -116,9 +116,6 @@ if (-not $isDirectALGo) { } } -# Apply Custom AL-Go System Files from settings -ApplyCustomALGoSystemFiles -path $templateFolder -settings $repoSettings - # CheckFiles is an array of hashtables with the following properties: # dstPath: The path to the file in the current repository # srcPath: The path to the file in the template repository @@ -261,6 +258,9 @@ foreach($checkfile in $checkfiles) { } } +# Apply Custom AL-Go System Files from settings +ApplyCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects + if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available if (($updateFiles) -or ($removeFiles)) { diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 3867f89be..f69ede4e2 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -150,16 +150,12 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings ``` "customALGoSystemFiles": [ { - "Destionation": ".AL-Go/", - "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1", + "Destination": ".AL-Go/", + "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1" }, { - "Destionation": ".AL-Go/", - "Source": "https://raw.githubusercontent.com/freddydk/CustomALGoSystemFiles/main/.AL-Go/myDevEnv.ps1", - }, - { - "Destionation": ".AL-Go/NewBcContainer.ps1", - "Source": "https://raw.githubusercontent.com/microsoft/BCApps/main/build/scripts/NewBcContainer.ps1", + "Destination": ".AL-Go/NewBcContainer.ps1", + "Source": "https://raw.githubusercontent.com/microsoft/BCApps/main/build/scripts/NewBcContainer.ps1" }, { "Destination": ".github/", From fd03c25165354ae69194385313aeebc68f955aee Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 06:59:41 +0200 Subject: [PATCH 059/108] add to updates files --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index ca19d166c..5e2a5cf7f 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -497,7 +497,7 @@ function ApplyCustomALGoSystemFiles { else { Write-Host "- $_" } - Copy-Item -Path $source -Destination $finalDestination + Write-Output @{ "DstFile" = $finalDestination; "content" = (Get-Content -Raw -Encoding UTF8 -Path $source) } } } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index cc9928ac4..8f987720a 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -259,7 +259,7 @@ foreach($checkfile in $checkfiles) { } # Apply Custom AL-Go System Files from settings -ApplyCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects +$updateFiles += @(ApplyCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects) if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available From 9ac3f24140066de8c32ac2c6e52dcca37417692f Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 07:11:31 +0200 Subject: [PATCH 060/108] relative --- .../CheckForUpdates.HelperFunctions.ps1 | 25 +++++-------------- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 5e2a5cf7f..8da651797 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -464,17 +464,16 @@ function UpdateSettingsFile { $settings | Set-JsonContentLF -path $settingsFile } -function ApplyCustomALGoSystemFiles { +function GetCustomALGoSystemFiles { Param( [string] $path, [hashtable] $settings, [string[]] $projects ) - function CopyItem{ + function YieldItem{ Param( [string] $source, - [string] $basePath, [string] $destination, [string[]] $projects ) @@ -486,18 +485,8 @@ function ApplyCustomALGoSystemFiles { $destinations = @($destination) } $destinations | ForEach-Object { - $finalDestination = Join-Path $basePath $_ - $finalDestinationFolder = [System.IO.Path]::GetDirectoryName($finalDestination) - if (!(Test-Path -Path $finalDestinationFolder -PathType Container)) { - New-Item -path $finalDestinationFolder -ItemType Directory | Out-Null - } - if (Test-Path $finalDestination) { - Write-Host "- $_ (overriding existing file)" - } - else { - Write-Host "- $_" - } - Write-Output @{ "DstFile" = $finalDestination; "content" = (Get-Content -Raw -Encoding UTF8 -Path $source) } + Write-Host "- $_" + Write-Output @{ "DstFile" = $_; "content" = (Get-Content -Raw -Encoding UTF8 -Path $source) } } } @@ -524,7 +513,6 @@ function ApplyCustomALGoSystemFiles { New-Item -Path $tempFolder -ItemType Directory | Out-Null $ext = [System.IO.Path]::GetExtension($source) $zipName = "$tempFolder$ext" - Push-Location -Path $path try { if ($ext -eq '.zip') { Write-Host "$($destination):" @@ -545,7 +533,7 @@ function ApplyCustomALGoSystemFiles { $destRelativeFileName = Resolve-Path $_.FullName -Relative $destFileName = Join-Path $destination $destRelativeFileName $destFileName = $destFileName.TrimStart('\/') - CopyItem -source $_.FullName -basePath $path -destination $destFileName -projects $projects + YieldItem -source $_.FullName -destination $destFileName -projects $projects } } finally { @@ -562,11 +550,10 @@ function ApplyCustomALGoSystemFiles { Write-Host "$($destination):" $tempFilename = Join-Path $tempFolder ([System.IO.Path]::GetFileName($source)) Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $tempFilename - CopyItem -source $tempFilename -basePath $path -destination $destination -projects $projects + YieldItem -source $tempFilename -destination $destination -projects $projects } } finally { - Pop-Location if (Test-Path -Path $zipName) { Remove-Item $zipName -Force } Remove-Item -Path $tempFolder -Recurse -Force } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 8f987720a..99b8f99f6 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -259,7 +259,9 @@ foreach($checkfile in $checkfiles) { } # Apply Custom AL-Go System Files from settings -$updateFiles += @(ApplyCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects) +$updateFiles += @(GetCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects) + +$updateFiles | Out-Host if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available From 362ec0f0ac7d8883f8a1701b9226a4aae8b6445b Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 07:12:14 +0200 Subject: [PATCH 061/108] remove path --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 1 - Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 8da651797..38770b9e9 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -466,7 +466,6 @@ function UpdateSettingsFile { function GetCustomALGoSystemFiles { Param( - [string] $path, [hashtable] $settings, [string[]] $projects ) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 99b8f99f6..560189afd 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -259,7 +259,7 @@ foreach($checkfile in $checkfiles) { } # Apply Custom AL-Go System Files from settings -$updateFiles += @(GetCustomALGoSystemFiles -path $baseFolder -settings $repoSettings -projects $projects) +$updateFiles += @(GetCustomALGoSystemFiles -settings $repoSettings -projects $projects) $updateFiles | Out-Host From 977f94b9f00dcee1eb1960ad9f831fc08312803e Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 07:18:00 +0200 Subject: [PATCH 062/108] do not dump custom algo system files --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 560189afd..772ad3332 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -261,8 +261,6 @@ foreach($checkfile in $checkfiles) { # Apply Custom AL-Go System Files from settings $updateFiles += @(GetCustomALGoSystemFiles -settings $repoSettings -projects $projects) -$updateFiles | Out-Host - if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available if (($updateFiles) -or ($removeFiles)) { From 4ccfc85250a76fe10d1ec953ae8c204c293e889f Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 07:36:35 +0200 Subject: [PATCH 063/108] compare --- .../CheckForUpdates.HelperFunctions.ps1 | 27 +++++++++++-------- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 38770b9e9..83c90d95b 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -466,6 +466,7 @@ function UpdateSettingsFile { function GetCustomALGoSystemFiles { Param( + [string] $baseFolder, [hashtable] $settings, [string[]] $projects ) @@ -485,10 +486,17 @@ function GetCustomALGoSystemFiles { } $destinations | ForEach-Object { Write-Host "- $_" - Write-Output @{ "DstFile" = $_; "content" = (Get-Content -Raw -Encoding UTF8 -Path $source) } + $content = Get-ContentLF -Path $source + if (Test-Path -Path $_) { + $existingContent = Get-ContentLF -Path $_ + } + if ($content -ne $existingContent) { + Write-Output @{ "DstFile" = $_; "content" = $content } + } } } + Push-Location $baseFolder if ($settings.customALGoSystemFiles -isnot [Array]) { throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." } @@ -527,17 +535,13 @@ function GetCustomALGoSystemFiles { Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve Push-Location -Path $subFolder - try { - Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { - $destRelativeFileName = Resolve-Path $_.FullName -Relative - $destFileName = Join-Path $destination $destRelativeFileName - $destFileName = $destFileName.TrimStart('\/') - YieldItem -source $_.FullName -destination $destFileName -projects $projects - } - } - finally { - Pop-Location + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + $destRelativeFileName = Resolve-Path $_.FullName -Relative + $destFileName = Join-Path $destination $destRelativeFileName + $destFileName = $destFileName.TrimStart('\/') + YieldItem -source $_.FullName -destination $destFileName -projects $projects } + Pop-Location } else { if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { @@ -557,4 +561,5 @@ function GetCustomALGoSystemFiles { Remove-Item -Path $tempFolder -Recurse -Force } } + Pop-Location } \ No newline at end of file diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 772ad3332..a5d5eaa8f 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -259,7 +259,7 @@ foreach($checkfile in $checkfiles) { } # Apply Custom AL-Go System Files from settings -$updateFiles += @(GetCustomALGoSystemFiles -settings $repoSettings -projects $projects) +$updateFiles += @(GetCustomALGoSystemFiles -baseFolder $baseFolder -settings $repoSettings -projects $projects) if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available From 48cd019e893f005ab7eccc3f43b87c6ebadb2733 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 09:12:23 +0200 Subject: [PATCH 064/108] define existing --- .../CheckForUpdates.HelperFunctions.ps1 | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 83c90d95b..6fb7f0e48 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -487,6 +487,7 @@ function GetCustomALGoSystemFiles { $destinations | ForEach-Object { Write-Host "- $_" $content = Get-ContentLF -Path $source + $existingContent = '' if (Test-Path -Path $_) { $existingContent = Get-ContentLF -Path $_ } @@ -535,13 +536,17 @@ function GetCustomALGoSystemFiles { Expand-Archive -Path $zipName -DestinationPath $tempFolder -Force $subFolder = Join-Path $tempFolder ([System.IO.Path]::GetDirectoryName($fileSpec)) -Resolve Push-Location -Path $subFolder - Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { - $destRelativeFileName = Resolve-Path $_.FullName -Relative - $destFileName = Join-Path $destination $destRelativeFileName - $destFileName = $destFileName.TrimStart('\/') - YieldItem -source $_.FullName -destination $destFileName -projects $projects + try { + Get-ChildItem -Path $subFolder -Filter ([System.IO.Path]::GetFileName($fileSpec)) -Recurse:$recurse -File | ForEach-Object { + $destRelativeFileName = Resolve-Path $_.FullName -Relative + $destFileName = Join-Path $destination $destRelativeFileName + $destFileName = $destFileName.TrimStart('\/') + YieldItem -source $_.FullName -destination $destFileName -projects $projects + } + } + finally { + Pop-Location } - Pop-Location } else { if ($customSpec.ContainsKey('FileSpec') -or $customSpec.ContainsKey('Recurse')) { From 5b70630da85cfe5fef9e7aa429da4ad79d6d0258 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 09:25:27 +0200 Subject: [PATCH 065/108] dump modified files --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index a5d5eaa8f..af83e2cad 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -264,6 +264,14 @@ $updateFiles += @(GetCustomALGoSystemFiles -baseFolder $baseFolder -settings $re if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available if (($updateFiles) -or ($removeFiles)) { + if ($updateFiles) { + Write-Host "Updated files:" + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" + } + if ($removeFiles) { + Write-Host "Removed files:" + $removeFiles | ForEach-Object { Write-Host "- $_" + } OutputWarning -message "There are updates for your AL-Go system, run 'Update AL-Go System Files' workflow to download the latest version of AL-Go." } else { From f83caa2afa39e22f6b48eb3c47c86710a3330160 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 09:30:58 +0200 Subject: [PATCH 066/108] syntax --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index af83e2cad..6b7aa0ffa 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -266,11 +266,11 @@ if ($update -ne 'Y') { if (($updateFiles) -or ($removeFiles)) { if ($updateFiles) { Write-Host "Updated files:" - $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" } } if ($removeFiles) { Write-Host "Removed files:" - $removeFiles | ForEach-Object { Write-Host "- $_" + $removeFiles | ForEach-Object { Write-Host "- $_" } } OutputWarning -message "There are updates for your AL-Go system, run 'Update AL-Go System Files' workflow to download the latest version of AL-Go." } From 7646df4481f469470030a04941160ca3c7bd7ff5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 09:42:40 +0200 Subject: [PATCH 067/108] use basefolder --- .../CheckForUpdates.HelperFunctions.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 6fb7f0e48..40efb10d3 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -473,6 +473,7 @@ function GetCustomALGoSystemFiles { function YieldItem{ Param( + [string] $baseFolder, [string] $source, [string] $destination, [string[]] $projects @@ -487,9 +488,10 @@ function GetCustomALGoSystemFiles { $destinations | ForEach-Object { Write-Host "- $_" $content = Get-ContentLF -Path $source + $existingFile = Join-Path $baseFolder $_ $existingContent = '' - if (Test-Path -Path $_) { - $existingContent = Get-ContentLF -Path $_ + if (Test-Path -Path $existingFile) { + $existingContent = Get-ContentLF -Path $existingFile } if ($content -ne $existingContent) { Write-Output @{ "DstFile" = $_; "content" = $content } @@ -497,7 +499,6 @@ function GetCustomALGoSystemFiles { } } - Push-Location $baseFolder if ($settings.customALGoSystemFiles -isnot [Array]) { throw "customALGoSystemFiles setting is wrongly formatted, must be an array of objects. See https://aka.ms/algosettings#customalgosystemfiles." } @@ -541,7 +542,7 @@ function GetCustomALGoSystemFiles { $destRelativeFileName = Resolve-Path $_.FullName -Relative $destFileName = Join-Path $destination $destRelativeFileName $destFileName = $destFileName.TrimStart('\/') - YieldItem -source $_.FullName -destination $destFileName -projects $projects + YieldItem -baseFolder $baseFolder -source $_.FullName -destination $destFileName -projects $projects } } finally { @@ -558,7 +559,7 @@ function GetCustomALGoSystemFiles { Write-Host "$($destination):" $tempFilename = Join-Path $tempFolder ([System.IO.Path]::GetFileName($source)) Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $tempFilename - YieldItem -source $tempFilename -destination $destination -projects $projects + YieldItem -baseFolder $baseFolder -source $tempFilename -destination $destination -projects $projects } } finally { @@ -566,5 +567,4 @@ function GetCustomALGoSystemFiles { Remove-Item -Path $tempFolder -Recurse -Force } } - Pop-Location } \ No newline at end of file From c23d28c4f166ac6ac4a03a07e07a1bb49a2d94a1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 10:02:34 +0200 Subject: [PATCH 068/108] doc --- Scenarios/CustomizingALGoForGitHub.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index f69ede4e2..e8ff18319 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -170,8 +170,8 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings | Property | Description | Mandatory | Default | | :-- | :-- | :-: | :-- | -| Destination | Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) | Yes | | -| Source | URL to a either a single file or a .zip file containing | Yes | | +| Destination | Path in which the file should be placed. Can include the filename if the source doesn't point to a .zip file, must include a terminating / or \\ if a filename is not included. | Yes | | +| Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. | Yes | | | FileSpec | If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | | Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | From c7c49d84e8f53cefb623947208510762a7e14fee Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 19 Aug 2024 14:49:58 +0200 Subject: [PATCH 069/108] upd --- Scenarios/CustomizingALGoForGitHub.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index e8ff18319..aa784793d 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -253,7 +253,7 @@ If any of your custom steps require permissions, which exceeds the permissions a ## Using indirect templates -If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. +If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. > \[!NOTE\] > Indirect templates can be public or private. @@ -261,8 +261,7 @@ If you have have customizations you want to apply to multiple repositories, you > \[!TIP\] -> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specied. - +> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. From f8a1310060c5a6c5e129600159def0a0a64fe5ae Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 07:33:02 +0200 Subject: [PATCH 070/108] update custom and unused algo system files --- .../CheckForUpdates.HelperFunctions.ps1 | 14 ++++++++++---- Actions/CheckForUpdates/CheckForUpdates.ps1 | 8 ++++++-- Scenarios/CustomizingALGoForGitHub.md | 3 +++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 40efb10d3..72d9e191c 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -434,7 +434,7 @@ function UpdateSettingsFile { Param( [string] $settingsFile, [hashtable] $updateSettings, - [hashtable] $additionalSettings = @{} + [hashtable] $indirectTemplateSettings = @{} ) # Update Repo Settings file with the template URL @@ -453,13 +453,19 @@ function UpdateSettingsFile { $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $updateSettings."$key" } } - # Grab settings from additionalSettings if they are not already in settings - foreach($key in $additionalSettings.Keys) { + # Grab settings from indirectTemplateSettings if they are not already in settings + foreach($key in $indirectTemplateSettings.Keys) { + # CustomALGoSystemFiles will not be copied from the indirect template settings - they will be applied to the indirect template + # UnusedALGoSystemFiles will not be copied from the indirect template settings - they will be used during the update process + if (@('customALGoSystemFiles','unusedALGoSystemFiles') -contains $key) { + continue + } if (!($settings.PSObject.Properties.Name -eq $key)) { # Add the property if it doesn't exist - $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $additionalSettings."$key" + $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $indirectTemplateSettings."$key" } } + # Save the file with LF line endings and UTF8 encoding $settings | Set-JsonContentLF -path $settingsFile } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 6b7aa0ffa..af09982e7 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -107,6 +107,10 @@ if (-not $isDirectALGo) { $templateOwner = $realTemplateUrl.Split('/')[3] $indirectTemplateRepoSettings = $templateRepoSettings + # If the indirect template contains unusedALGoSystemFiles, we need to remove them from the current repository + if ($indirectTemplateRepoSettings.ContainsKey('unusedALGoSystemFiles')) { + $unusedALGoSystemFiles += $indirectTemplateRepoSettings.unusedALGoSystemFiles + } $myALGoSettingsFile = Join-Path $templateFolder "*/$ALGoSettingsFile" if (Test-Path $myALGoSettingsFile -PathType Leaf) { Write-Host "Read project settings from indirect template repository" @@ -175,7 +179,7 @@ foreach($checkfile in $checkfiles) { Write-Host "Update Project Settings" # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) $projectSettingsFile = Join-Path $dstFolder "settings.json" - UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -additionalSettings $indirectTemplateProjectSettings + UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -indirectTemplateSettings $indirectTemplateProjectSettings $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } } @@ -295,7 +299,7 @@ else { invoke-git status - UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -additionalSettings $indirectTemplateRepoSettings + UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -indirectTemplateSettings $indirectTemplateRepoSettings # Update the files # Calculate the release notes, while updating diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index aa784793d..acfd9854e 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -258,7 +258,10 @@ If you have have customizations you want to apply to multiple repositories, you > \[!NOTE\] > Indirect templates can be public or private. +Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. +> \[!NOTE\] +> CustomALGoSystemFiles will NOT be copied from the indirect template, they will be applied there and UnusedALGoSystemFiles will be merged. > \[!TIP\] > The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. From 4317c17740e68cdf67150f812d43fd5ebdbfc305 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 07:56:26 +0200 Subject: [PATCH 071/108] Support private template repo --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 5 +++++ Templates/AppSource App/.github/workflows/CICD.yaml | 9 +++++++++ .../Per Tenant Extension/.github/workflows/CICD.yaml | 9 +++++++++ 3 files changed, 23 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index af09982e7..e736eebdc 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -30,6 +30,11 @@ if ($update -eq 'Y') { $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } } +else { + if (-not $token) { + $token = $env:GITHUB_TOKEN + } +} # Use Authenticated API request to avoid the 60 API calls per hour limit $headers = @{ diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index c6f19ed66..851f956ed 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -134,10 +134,19 @@ jobs: shell: powershell get: templateUrl + - name: Read secrets + id: ReadSecrets + uses: microsoft/AL-Go-Actions/ReadSecrets@main + with: + shell: powershell + gitHubSecrets: ${{ toJson(secrets) }} + getSecrets: 'ghTokenWorkflow' + - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 0dc128768..1ee6dc260 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -134,10 +134,19 @@ jobs: shell: powershell get: templateUrl + - name: Read secrets + id: ReadSecrets + uses: microsoft/AL-Go-Actions/ReadSecrets@main + with: + shell: powershell + gitHubSecrets: ${{ toJson(secrets) }} + getSecrets: 'ghTokenWorkflow' + - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true From 57acb7cc481cd1d023186a07a17000dd74349626 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 08:14:05 +0200 Subject: [PATCH 072/108] use ghtokenworkflow --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 10 ++++------ Actions/CheckForUpdates/action.yaml | 2 +- Scenarios/CustomizingALGoForGitHub.md | 4 ++-- .../AppSource App/.github/workflows/CreateRelease.yaml | 3 ++- .../.github/workflows/CreateRelease.yaml | 3 ++- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index e736eebdc..2c87853de 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -22,18 +22,16 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper -if ($update -eq 'Y') { - if (-not $token) { +if (-not $token) { + if ($update -eq 'Y') { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." } else { - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) + $token = $env:GITHUB_TOKEN } } else { - if (-not $token) { - $token = $env:GITHUB_TOKEN - } + $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } # Use Authenticated API request to avoid the 60 API calls per hour limit diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index 4b12bc6f7..28ea8e24e 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -16,7 +16,7 @@ inputs: token: description: The GitHub token running the action required: false - default: ${{ github.token }} + default: '' templateUrl: description: URL of the template repository (default is the template repository used to create the repository) required: false diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index acfd9854e..5badcc887 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -256,12 +256,12 @@ If any of your custom steps require permissions, which exceeds the permissions a If you have have customizations you want to apply to multiple repositories, you might want to consider using an indirect template. An indirect template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. > \[!NOTE\] -> Indirect templates can be public or private. +> Indirect templates can be public or private. If you are using a private indirect template, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. > \[!NOTE\] -> CustomALGoSystemFiles will NOT be copied from the indirect template, they will be applied there and UnusedALGoSystemFiles will be merged. +> UnusedALGoSystemFiles and CustomALGoSystemFiles will NOT be copied from the indirect template, they will be applied during Update AL-Go System Files. > \[!TIP\] > The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index a69bbe356..c3d74ff85 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -99,7 +99,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'TokenForPush' + getSecrets: 'TokenForPush,GhTokenWorkflow' useGhTokenWorkflowForPush: '${{ github.event.inputs.useGhTokenWorkflow }}' - name: Determine Projects @@ -112,6 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index 5cb6d41e4..b84620364 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -99,7 +99,7 @@ jobs: with: shell: powershell gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: 'TokenForPush' + getSecrets: 'TokenForPush,GhTokenWorkflow' useGhTokenWorkflowForPush: '${{ github.event.inputs.useGhTokenWorkflow }}' - name: Determine Projects @@ -112,6 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true From 557350608d1c08ee7bc98fa80dde1f5a514b312c Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 08:20:00 +0200 Subject: [PATCH 073/108] dump content --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 2c87853de..986deda7a 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -273,7 +273,8 @@ if ($update -ne 'Y') { if (($updateFiles) -or ($removeFiles)) { if ($updateFiles) { Write-Host "Updated files:" - $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" } + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)"; Write-Host "'$($_.Content)'" } + } if ($removeFiles) { Write-Host "Removed files:" From 3ec676b913d580d7e05ce2a06a5857a9f9534908 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 08:25:47 +0200 Subject: [PATCH 074/108] only update changed settings files --- .../CheckForUpdates.HelperFunctions.ps1 | 16 +++++++++++++--- Actions/CheckForUpdates/CheckForUpdates.ps1 | 12 ++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 72d9e191c..11a29212b 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -437,20 +437,26 @@ function UpdateSettingsFile { [hashtable] $indirectTemplateSettings = @{} ) + $modified = $false # Update Repo Settings file with the template URL if (Test-Path $settingsFile) { $settings = Get-Content $settingsFile -Encoding UTF8 | ConvertFrom-Json } else { $settings = [PSCustomObject]@{} + $modified = $true } foreach($key in $updateSettings.Keys) { if ($settings.PSObject.Properties.Name -eq $key) { - $settings."$key" = $updateSettings."$key" + if ($settings."$key" -ne $updateSettings."$key") { + $settings."$key" = $updateSettings."$key" + $modified = $true + } } else { # Add the property if it doesn't exist $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $updateSettings."$key" + $modified = $true } } # Grab settings from indirectTemplateSettings if they are not already in settings @@ -463,11 +469,15 @@ function UpdateSettingsFile { if (!($settings.PSObject.Properties.Name -eq $key)) { # Add the property if it doesn't exist $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $indirectTemplateSettings."$key" + $modified = $true } } - # Save the file with LF line endings and UTF8 encoding - $settings | Set-JsonContentLF -path $settingsFile + if ($modified) { + # Save the file with LF line endings and UTF8 encoding + $settings | Set-JsonContentLF -path $settingsFile + } + return $modified } function GetCustomALGoSystemFiles { diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 986deda7a..2114c23be 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -182,8 +182,9 @@ foreach($checkfile in $checkfiles) { Write-Host "Update Project Settings" # Copy individual settings from the indirect template repository .AL-Go/settings.json (if the setting doesn't exist in the project folder) $projectSettingsFile = Join-Path $dstFolder "settings.json" - UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -indirectTemplateSettings $indirectTemplateProjectSettings - $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } + if (UpdateSettingsFile -settingsFile $projectSettingsFile -updateSettings @{} -indirectTemplateSettings $indirectTemplateProjectSettings) { + $updateFiles += @{ "DstFile" = Join-Path $dstPath "settings.json"; "content" = (Get-Content -Path $projectSettingsFile -Encoding UTF8 -Raw) } + } } # Remove unused AL-Go system files @@ -273,7 +274,7 @@ if ($update -ne 'Y') { if (($updateFiles) -or ($removeFiles)) { if ($updateFiles) { Write-Host "Updated files:" - $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)"; Write-Host "'$($_.Content)'" } + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" } } if ($removeFiles) { @@ -303,7 +304,10 @@ else { invoke-git status - UpdateSettingsFile -settingsFile (Join-Path ".github" "AL-Go-Settings.json") -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -indirectTemplateSettings $indirectTemplateRepoSettings + $repoSettingsFile = Join-Path ".github" "AL-Go-Settings.json" + if (UpdateSettingsFile -settingsFile $repoSettingsFile -updateSettings @{ "templateUrl" = $templateUrl; "templateSha" = $templateSha } -indirectTemplateSettings $indirectTemplateRepoSettings) { + $updateFiles += @{ "DstFile" = $repoSettingsFile; "content" = (Get-Content -Path $repoSettingsFile -Encoding UTF8 -Raw) } + } # Update the files # Calculate the release notes, while updating From 0d39e968aebee2c236c1c5efc126fb240e54b57b Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 08:56:58 +0200 Subject: [PATCH 075/108] decode --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 2114c23be..b1bf85328 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -31,6 +31,7 @@ if (-not $token) { } } else { + # Specified token is GhTokenWorkflow secret - decode from base 64 $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } From 3451a4a24cd827db6cc181e967fe6416043fe3aa Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:11:03 +0200 Subject: [PATCH 076/108] indirect --- Scenarios/CustomizingALGoForGitHub.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 5badcc887..3aa08539f 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -258,22 +258,21 @@ If you have have customizations you want to apply to multiple repositories, you > \[!NOTE\] > Indirect templates can be public or private. If you are using a private indirect template, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. -Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. - -> \[!NOTE\] -> UnusedALGoSystemFiles and CustomALGoSystemFiles will NOT be copied from the indirect template, they will be applied during Update AL-Go System Files. +Repository and project settings from the indirect template will also be applied to the new repository during update AL-Go System Files, unless the setting already exists in the repository being updated. **UnusedALGoSystemFiles** and **CustomALGoSystemFiles** will NOT be copied from the indirect template, they will be applied during Update AL-Go System Files. > \[!TIP\] -> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a GhTokenWorkflow secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. - - +> The recommended way to create a new repository based on your indirect AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a **GhTokenWorkflow** secret and then run the `Update AL-Go System Files` workflow with your indirect template specified. > \[!NOTE\] -> If you use the indirect template as a GitHub template, for creating the repository, then you need to re-specify the indirect Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the indirect template as it's template repository. - +> If you use the indirect template as a GitHub template for creating the repository, by clicking use this template in your indirect template - then you need to re-specify the indirect Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the indirect template as it's template repository. +Repositories based on your indirect template will notify you that changes are available for your AL-Go System Files when you update the indirect template only. You will not be notified when new versions of AL-Go for GitHub is released in every repository - only in the indirect template repository. +> \[!WARNING\] +> You should ensure that your indirect template repository is kept up-to-date with the latest changes in AL-Go for GitHub. +> \[!TIP\] +> You can setup the Update AL-Go System Files workflow to run on a schedule to uptake new releases of AL-Go for GitHub regularly. ## Forking AL-Go for GitHub and making your "own" **public** version From 37b7ecbfa5e6f72e346b205bf879a42f4ad57971 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:12:46 +0200 Subject: [PATCH 077/108] insert --- .github/workflows/jekyll-gh-pages.yml | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml new file mode 100644 index 000000000..ded8688cc --- /dev/null +++ b/.github/workflows/jekyll-gh-pages.yml @@ -0,0 +1,49 @@ +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Deploy Jekyll with GitHub Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Setup Pages + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + - name: Build with Jekyll + uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 + with: + source: . + destination: ./_site + - name: Upload artifact + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 + with: + preview: true \ No newline at end of file From dccb8e4f78f2609ec0070f26b23b653adaf47ba1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:14:26 +0200 Subject: [PATCH 078/108] remove jekyll --- .github/workflows/jekyll-gh-pages.yml | 49 --------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml deleted file mode 100644 index ded8688cc..000000000 --- a/.github/workflows/jekyll-gh-pages.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Jekyll with GitHub Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 - with: - source: . - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 - with: - preview: true \ No newline at end of file From 570628a7a293dc41eb64da1ac64470be4eef7cb8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:20:00 +0200 Subject: [PATCH 079/108] revert --- .github/workflows/powershell.yaml | 2 +- .../.github/workflows/_BuildALGoProject.yaml | 20 +++++++++---------- .../.github/workflows/_BuildALGoProject.yaml | 20 +++++++++---------- .../_BuildPowerPlatformSolution.yaml | 2 +- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/powershell.yaml b/.github/workflows/powershell.yaml index 6fe36a18e..5a7883ad4 100644 --- a/.github/workflows/powershell.yaml +++ b/.github/workflows/powershell.yaml @@ -31,6 +31,6 @@ jobs: # Upload the SARIF file generated in the previous step - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15 + uses: github/codeql-action/upload-sarif@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 with: sarif_file: results.sarif diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index a875964ff..9f0fa2b44 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -168,7 +168,7 @@ jobs: - name: Upload thisbuild artifacts - apps if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildAppsArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/Apps/' @@ -177,7 +177,7 @@ jobs: - name: Upload thisbuild artifacts - dependencies if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildDependenciesArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/Dependencies/' @@ -186,7 +186,7 @@ jobs: - name: Upload thisbuild artifacts - test apps if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildTestAppsArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/TestApps/' @@ -194,7 +194,7 @@ jobs: retention-days: 1 - name: Publish artifacts - apps - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.AppsArtifactsName }} @@ -202,7 +202,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - dependencies - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts && env.generateDependencyArtifact == 'True' with: name: ${{ steps.calculateArtifactsNames.outputs.DependenciesArtifactsName }} @@ -210,7 +210,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - test apps - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.TestAppsArtifactsName }} @@ -218,7 +218,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - build output - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/BuildOutput.txt',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.BuildOutputArtifactsName }} @@ -226,7 +226,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - container event log - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (failure()) && (hashFiles(format('{0}/ContainerEventLog.evtx',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.ContainerEventLogArtifactsName }} @@ -234,7 +234,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - test results - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/TestResults.xml',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.TestResultsArtifactsName }} @@ -242,7 +242,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - bcpt test results - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/bcptTestResults.json',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.BcptTestResultsArtifactsName }} diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index a875964ff..9f0fa2b44 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -168,7 +168,7 @@ jobs: - name: Upload thisbuild artifacts - apps if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildAppsArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/Apps/' @@ -177,7 +177,7 @@ jobs: - name: Upload thisbuild artifacts - dependencies if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildDependenciesArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/Dependencies/' @@ -186,7 +186,7 @@ jobs: - name: Upload thisbuild artifacts - test apps if: inputs.publishThisBuildArtifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: ${{ steps.calculateArtifactsNames.outputs.ThisBuildTestAppsArtifactsName }} path: '${{ inputs.project }}/.buildartifacts/TestApps/' @@ -194,7 +194,7 @@ jobs: retention-days: 1 - name: Publish artifacts - apps - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.AppsArtifactsName }} @@ -202,7 +202,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - dependencies - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts && env.generateDependencyArtifact == 'True' with: name: ${{ steps.calculateArtifactsNames.outputs.DependenciesArtifactsName }} @@ -210,7 +210,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - test apps - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.TestAppsArtifactsName }} @@ -218,7 +218,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - build output - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/BuildOutput.txt',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.BuildOutputArtifactsName }} @@ -226,7 +226,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - container event log - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (failure()) && (hashFiles(format('{0}/ContainerEventLog.evtx',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.ContainerEventLogArtifactsName }} @@ -234,7 +234,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - test results - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/TestResults.xml',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.TestResultsArtifactsName }} @@ -242,7 +242,7 @@ jobs: if-no-files-found: ignore - name: Publish artifacts - bcpt test results - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: (success() || failure()) && (hashFiles(format('{0}/bcptTestResults.json',inputs.project)) != '') with: name: ${{ steps.calculateArtifactsNames.outputs.BcptTestResultsArtifactsName }} diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml index bcf6e5479..d49bc9f3e 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml @@ -92,7 +92,7 @@ jobs: suffix: ${{ inputs.artifactsNameSuffix }} - name: Publish artifacts - Power Platform Solution - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 if: inputs.publishArtifacts with: name: ${{ steps.calculateArtifactsNames.outputs.PowerPlatformSolutionArtifactsName }} From e599e3873415ceae163632b930d07c5b033e9bc1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:20:54 +0200 Subject: [PATCH 080/108] update --- .github/workflows/powershell.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/powershell.yaml b/.github/workflows/powershell.yaml index 5a7883ad4..04885b056 100644 --- a/.github/workflows/powershell.yaml +++ b/.github/workflows/powershell.yaml @@ -31,6 +31,6 @@ jobs: # Upload the SARIF file generated in the previous step - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 + uses: github/codeql-action/upload-sarif@429e1977040da7a23b6822b13c129cd1ba93dbb2 # v3.26.2 with: sarif_file: results.sarif From 5cc070b3c4b671586bf20aaf7a0ef2e382addfa2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:27:39 +0200 Subject: [PATCH 081/108] update release notes --- Actions/ReadSecrets/ReadSecretsHelper.psm1 | 11 ++++- RELEASENOTES.md | 57 ++++++++++++---------- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/Actions/ReadSecrets/ReadSecretsHelper.psm1 b/Actions/ReadSecrets/ReadSecretsHelper.psm1 index caa10acd0..4f7b48044 100644 --- a/Actions/ReadSecrets/ReadSecretsHelper.psm1 +++ b/Actions/ReadSecrets/ReadSecretsHelper.psm1 @@ -26,7 +26,6 @@ function GetAzureCredentials { return $null } - function MaskValue { Param( [string] $key, @@ -119,6 +118,16 @@ function GetKeyVaultCredentials { $creds = $null } } + if ($creds) { + try { + # check that we have access to get secrets from the keyvault by trying to get a dummy secret + GetKeyVaultSecret -secretName 'algodummysecret' -keyVaultCredentials $creds -encrypted | Out-Null + } + catch { + Write-Host "Unable to get secrets from Azure Key Vault. Error was $($_.Exception.Message). Using Github secrets instead." + $creds = $null + } + } return $creds } diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 9abaac27e..55490330f 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,36 @@ ### Issues +### New Settings + +- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: + + - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) + - **Source** (mandatory) - URL to a either a single file or a .zip file containing + - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. + - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) + +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Add custom steps to the AL-Go build workflow + +It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. + +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Indirect AL-Go template repository + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + +## v5.3 + +### Issues + - Issue 1105 Increment Version Number - repoVersion in .github/AL-Go-Settings.json is not updated - Issue 1073 Publish to AppSource - Automated validation: failure - Issue 980 Allow Scope to be PTE in continuousDeployment for PTE extensions in Sandbox (enhancement request) @@ -10,6 +41,7 @@ - Fix issue with github ref when running reusable workflows - Issue 1098 Support for specifying the name of the AZURE_CREDENTIALS secret by adding a AZURE_CREDENTIALSSecretName setting - Fix placeholder syntax for git ref in PullRequestHandler.yaml +- Issue 1164 Getting secrets from Azure key vault fails in Preview ### Dependencies to PowerShell modules @@ -47,34 +79,9 @@ AL-Go for GitHub now includes a new telemetry module. For detailed information o - **NumberOfSqlStmtsWarning** - a warning is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 5) - **NumberOfSqlStmtsError** - an error is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 10) -- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: - - - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) - - **Source** (mandatory) - URL to a either a single file or a .zip file containing - - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. - - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) - > \[!NOTE\] > Duration thresholds are subject to varying results depending on the performance of the agent running the tests. Number of SQL statements executed by a test is often the most reliable indicator of performance degredation. -### Add custom jobs to AL-Go workflows - -It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. - -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Add custom steps to the AL-Go build workflow - -It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. - -Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Indirect AL-Go template repository - -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. - -Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. - ## v5.2 ### Issues From cde82c9ce5a7d89b52d4ef1b81f08b2d8871f806 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:29:03 +0200 Subject: [PATCH 082/108] rev --- RELEASENOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 55490330f..74551a3ba 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -212,7 +212,7 @@ If false, the templateSha repository setting is used to download specific AL-Go ### New Settings -- `[alDoc](https://aka.ms/algosettings#aldoc)` : JSON object with properties for the ALDoc reference document generation +- `alDoc` : JSON object with properties for the ALDoc reference document generation - **continuousDeployment** = Determines if reference documentation will be deployed continuously as part of CI/CD. You can run the **Deploy Reference Documentation** workflow to deploy manually or on a schedule. (Default false) - **deployToGitHubPages** = Determines whether or not the reference documentation site should be deployed to GitHub Pages for the repository. In order to deploy to GitHub Pages, GitHub Pages must be enabled and set to GitHub Actions. (Default true) - **maxReleases** = Maximum number of releases to include in the reference documentation. (Default 3) From d60be053389f36bcc8e76cf17c444f670253ab44 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:40:21 +0200 Subject: [PATCH 083/108] psa --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 11a29212b..ea51d358c 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -494,7 +494,7 @@ function GetCustomALGoSystemFiles { [string] $destination, [string[]] $projects ) - + if ($destination -like ".AL-Go$([IO.Path]::DirectorySeparatorChar)*") { $destinations = $projects | ForEach-Object { Join-Path $_ $destination } } From fa4a71bb17f27e5e59ac56b1adbd393802941b2c Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:43:19 +0200 Subject: [PATCH 084/108] remove --- RELEASENOTES.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 3fce26636..74551a3ba 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,3 @@ -## v5.3 - ### Issues ### New Settings From 6685835c9676489effbb9c9dd4fb68c7361e5178 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:46:15 +0200 Subject: [PATCH 085/108] move --- .../.github/workflows/UpdateGitHubGoSystemFiles.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 9ded4de35..bfdf73edd 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -65,10 +65,10 @@ jobs: - name: Override templateUrl env: - templateUrl: ${{ github.event.inputs.templateUrl }} + newTemplateUrl: ${{ github.event.inputs.templateUrl }} run: | $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 - $templateUrl = $ENV:templateUrl + $templateUrl = $ENV:newTemplateUrl if ($templateUrl) { Write-Host "Using Template Url: $templateUrl" Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "templateUrl=$templateUrl" From 99265e6e4dec11ac8e44f1abe13fd9eb7f448b37 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:48:14 +0200 Subject: [PATCH 086/108] remove fake button --- Internal/Deploy.ps1 | 8 -------- Templates/AppSource App/README.md | 2 -- Templates/Per Tenant Extension/README.md | 2 -- 3 files changed, 12 deletions(-) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 122cab4ee..c6a9e05d6 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -252,14 +252,6 @@ try { } Set-Content -Path (Join-Path "./.github" "RELEASENOTES.copy.md") -Value $releaseNotes -Encoding utf8 } - - # Replace template_owner in README.md - $readmeFile = './README.md' - $readme = (Get-Content -Encoding utf8 -Path $readmeFile) -join "`n" - $readme = $readme.Replace('&template_owner=microsoft)', "&template_owner=$($config.githubOwner))") - Set-Content -Path $readmeFile -Encoding utf8 -Value $readme - - # Push changes PushChanges -BaseBranch $branch -CommitMessage "Deploying AL-Go from $algoBranch ($srcSHA) to $branch" -DirectCommit $directCommit } diff --git a/Templates/AppSource App/README.md b/Templates/AppSource App/README.md index ab1063e63..ea984de77 100644 --- a/Templates/AppSource App/README.md +++ b/Templates/AppSource App/README.md @@ -2,8 +2,6 @@ This template repository can be used for managing AppSource Apps for Business Central. -[![Use this template](https://github.com/microsoft/AL-Go/assets/10775043/ca1ecc85-2fd3-4ab5-a866-bd2e7e80259d)](https://github.com/new?template_name=AL-Go-AppSource&template_owner=microsoft) - Please go to https://aka.ms/AL-Go to learn more. ## Contributing diff --git a/Templates/Per Tenant Extension/README.md b/Templates/Per Tenant Extension/README.md index 81dd89505..45baaa734 100644 --- a/Templates/Per Tenant Extension/README.md +++ b/Templates/Per Tenant Extension/README.md @@ -2,8 +2,6 @@ This template repository can be used for managing AppSource Apps for Business Central. -[![Use this template](https://github.com/microsoft/AL-Go/assets/10775043/ca1ecc85-2fd3-4ab5-a866-bd2e7e80259d)](https://github.com/new?template_name=AL-Go-PTE&template_owner=microsoft) - Please go to https://aka.ms/AL-Go to learn more. ## Contributing From 0969502bf96c62df795b65b21dbcea4ce6f25182 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:50:16 +0200 Subject: [PATCH 087/108] use preview --- Actions/AL-Go-Helper.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 73c731ad9..5f98c53e8 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -18,7 +18,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent -$defaultBcContainerHelperVersion = "https://github.com/freddydk/navcontainerhelper/archive/refs/heads/runpipeline.zip" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" +$defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" $notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","EnvironmentName") $runAlPipelineOverrides = @( From a2dbcfdcdde03e6827e037a98fd984ee1c408bb7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 09:55:05 +0200 Subject: [PATCH 088/108] precommit --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- Scenarios/CustomizingALGoForGitHub.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index ea51d358c..b6f8a68c0 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -583,4 +583,4 @@ function GetCustomALGoSystemFiles { Remove-Item -Path $tempFolder -Recurse -Force } } -} \ No newline at end of file +} diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 3aa08539f..f6095e957 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -14,7 +14,7 @@ Microsoft will continuously develop and maintain AL-Go for GitHub and ensure tha Keeping your repositories up-to-date can be done manually or on a schedule (like Windows update really). You will be notified when an update is available and we recommend that you keep your repositories up-to-date at all time. If you make modifications to the AL-Go System Files (scripts and workflows) in your repository, in other ways than described in this document, these changes will be removed with the next AL-Go update. -> \[!TIP] +> \[!TIP\] > If for some reason the updated version of AL-Go for GitHub doesn't work for you, we recommend that you file an issue [here](https://github.com/microsoft/AL-Go/issues) with a detailed description of the problem and full logs of the failing workflows. You can then revert back to the prior version of AL-Go for GitHub until the issue is resolved. > > It is important to get back to the mainstream version of AL-Go for GitHub as soon as the issue is resolved. @@ -222,13 +222,13 @@ At this time, the anchor-points where you can add custom steps are: | Workflow | Job | Step | Before or AFter | | :-- | :-- | :-- | :-: | -| _BuildALGoProject.yaml | BuildALGoProject | Read settings | After | +| \_BuildALGoProject.yaml | BuildALGoProject | Read settings | After | | | | Read secrets | After | | | | Build | Before | | | | Read secrets | After | | | | Cleanup | Before | -The custom step needs to be named `CustomStep` and if inserted in any of the specified anchor-points, it will be maintained after running Update AL-Go System Files. An example of a custom step could be a step, which modifies settings based on some business logic +The custom step needs to be named `CustomStep` and if inserted in any of the specified anchor-points, it will be maintained after running Update AL-Go System Files. An example of a custom step could be a step, which modifies settings based on some business logic ```yaml - name: CustomStep-ModifySettings From a01e57addc0fe9b24aceb40f8b223d1e568241a2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:10:06 +0200 Subject: [PATCH 089/108] dump token length --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index b1bf85328..5e1a27208 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -28,6 +28,7 @@ if (-not $token) { } else { $token = $env:GITHUB_TOKEN + Write-Host $token.Length } } else { From 46da82c8433ad268924088bc4c369c3429dfaad9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:21:30 +0200 Subject: [PATCH 090/108] add ghTokenWorkflow parameter --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 18 +++++++----------- Actions/CheckForUpdates/action.yaml | 4 ++++ .../AppSource App/.github/workflows/CICD.yaml | 2 +- .../.github/workflows/CreateRelease.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- .../.github/workflows/CICD.yaml | 2 +- .../.github/workflows/CreateRelease.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 5e1a27208..43e33e90e 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -3,6 +3,8 @@ [string] $actor, [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] [string] $token, + [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] + [string] $ghTokenWorkflow, [Parameter(HelpMessage = "URL of the template repository (default is the template repository used to create the repository)", Mandatory = $false)] [string] $templateUrl = "", [Parameter(HelpMessage = "Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update)", Mandatory = $true)] @@ -22,18 +24,12 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper -if (-not $token) { - if ($update -eq 'Y') { - throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." - } - else { - $token = $env:GITHUB_TOKEN - Write-Host $token.Length - } -} -else { +if ($ghTokenWorkflow) { # Specified token is GhTokenWorkflow secret - decode from base 64 - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) + $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ghTokenWorkflow)) +} +elseif ($update -eq 'Y') { + throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." } # Use Authenticated API request to avoid the 60 API calls per hour limit diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index 28ea8e24e..18ff727fb 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -16,6 +16,10 @@ inputs: token: description: The GitHub token running the action required: false + default: ${{ github.token }} + ghTokenWorkflow: + description: Base64 encoded GhTokenWorkflow secret + required: false default: '' templateUrl: description: URL of the template repository (default is the template repository used to create the repository) diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index 851f956ed..cf4c0882a 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -146,7 +146,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index c3d74ff85..7d0410947 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -112,7 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index bfdf73edd..9c2f857f7 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -96,7 +96,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ env.templateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 1ee6dc260..c7f158d0a 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -146,7 +146,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index b84620364..c10a981b6 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -112,7 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index bfdf73edd..9c2f857f7 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -96,7 +96,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ env.templateUrl }} From fa5a2d43f34e7401ecf94b73346ab798be816cf5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:25:09 +0200 Subject: [PATCH 091/108] add ght --- Actions/CheckForUpdates/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index 18ff727fb..e799558ad 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -48,6 +48,7 @@ runs: env: _actor: ${{ inputs.actor }} _token: ${{ inputs.token }} + _ghTokenWorkflow: ${{ inputs.ghTokenWorkflow }} _templateUrl: ${{ inputs.templateUrl }} _downloadLatest: ${{ inputs.downloadLatest }} _update: ${{ inputs.update }} @@ -55,7 +56,7 @@ runs: _directCommit: ${{ inputs.directCommit }} run: | ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "CheckForUpdates" -Action { - ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') + ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -ghTokenWorkflow $ENV:_ghTokenWorkflow -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') } branding: icon: terminal From 5c40c1516a89abf4a1e318c2fd48f8f5a5c2e6ea Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:37:30 +0200 Subject: [PATCH 092/108] use token --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 19 ++++++++++++------- Actions/CheckForUpdates/README.md | 3 ++- Actions/CheckForUpdates/action.yaml | 12 ++++++------ .../AppSource App/.github/workflows/CICD.yaml | 2 +- .../.github/workflows/CreateRelease.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- .../.github/workflows/CICD.yaml | 2 +- .../.github/workflows/CreateRelease.yaml | 2 +- .../workflows/UpdateGitHubGoSystemFiles.yaml | 2 +- 9 files changed, 26 insertions(+), 20 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 43e33e90e..952a2dfbc 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -1,10 +1,10 @@ Param( [Parameter(HelpMessage = "The GitHub actor running the action", Mandatory = $false)] [string] $actor, - [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] - [string] $token, [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] - [string] $ghTokenWorkflow, + [string] $token, + [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] + [string] $githubToken, [Parameter(HelpMessage = "URL of the template repository (default is the template repository used to create the repository)", Mandatory = $false)] [string] $templateUrl = "", [Parameter(HelpMessage = "Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update)", Mandatory = $true)] @@ -24,12 +24,17 @@ # ContainerHelper is used for determining project folders and dependencies DownloadAndImportBcContainerHelper -if ($ghTokenWorkflow) { +if ($token) { # Specified token is GhTokenWorkflow secret - decode from base 64 - $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ghTokenWorkflow)) + $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } -elseif ($update -eq 'Y') { - throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." +else { + if ($update -eq 'Y') { + throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." + } + else { + $token = $githubToken + } } # Use Authenticated API request to avoid the 60 API calls per hour limit diff --git a/Actions/CheckForUpdates/README.md b/Actions/CheckForUpdates/README.md index 554ca9db5..c3db39db0 100644 --- a/Actions/CheckForUpdates/README.md +++ b/Actions/CheckForUpdates/README.md @@ -14,7 +14,8 @@ none | :-- | :-: | :-- | :-- | | shell | | The shell (powershell or pwsh) in which the PowerShell script in this action should run | powershell | | actor | | The GitHub actor running the action | github.actor | -| token | | The GitHub token running the action | github.token | +| token | | Base64 encoded GhTokenWorkflow secret | | +| githubToken | | The GitHub token running the action | github.token | | templateUrl | | URL of the template repository (default is the template repository used to create the repository) | default | | downloadLatest | Yes | Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update) | | | update | | Set this input to Y in order to update AL-Go System Files if needed | N | diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index e799558ad..b1dafdad6 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -14,13 +14,13 @@ inputs: required: false default: ${{ github.actor }} token: - description: The GitHub token running the action - required: false - default: ${{ github.token }} - ghTokenWorkflow: description: Base64 encoded GhTokenWorkflow secret required: false default: '' + githubToken: + description: The GitHub token running the action + required: false + default: ${{ github.token }} templateUrl: description: URL of the template repository (default is the template repository used to create the repository) required: false @@ -48,7 +48,7 @@ runs: env: _actor: ${{ inputs.actor }} _token: ${{ inputs.token }} - _ghTokenWorkflow: ${{ inputs.ghTokenWorkflow }} + _githubToken: ${{ inputs.githubToken }} _templateUrl: ${{ inputs.templateUrl }} _downloadLatest: ${{ inputs.downloadLatest }} _update: ${{ inputs.update }} @@ -56,7 +56,7 @@ runs: _directCommit: ${{ inputs.directCommit }} run: | ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "CheckForUpdates" -Action { - ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -ghTokenWorkflow $ENV:_ghTokenWorkflow -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') + ${{ github.action_path }}/CheckForUpdates.ps1 -actor $ENV:_actor -token $ENV:_token -githubToken $ENV:_githubToken -templateUrl $ENV:_templateUrl -downloadLatest ($ENV:_downloadLatest -eq 'true') -update $ENV:_update -updateBranch $ENV:_updateBranch -directCommit ($ENV:_directCommit -eq 'true') } branding: icon: terminal diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index cf4c0882a..851f956ed 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -146,7 +146,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index 7d0410947..c3d74ff85 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -112,7 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 9c2f857f7..bfdf73edd 100644 --- a/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/AppSource App/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -96,7 +96,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ env.templateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index c7f158d0a..1ee6dc260 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -146,7 +146,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index c10a981b6..b84620364 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -112,7 +112,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} templateUrl: ${{ env.templateUrl }} downloadLatest: true diff --git a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml index 9c2f857f7..bfdf73edd 100644 --- a/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/UpdateGitHubGoSystemFiles.yaml @@ -96,7 +96,7 @@ jobs: uses: microsoft/AL-Go-Actions/CheckForUpdates@main with: shell: powershell - ghTokenWorkflow: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} + token: ${{ fromJson(steps.ReadSecrets.outputs.Secrets).ghTokenWorkflow }} downloadLatest: ${{ env.downloadLatest }} update: 'Y' templateUrl: ${{ env.templateUrl }} From 000ae79c8bf638d58320efcfb323485c402c1e4f Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 10:47:43 +0200 Subject: [PATCH 093/108] dump with token is used --- Actions/CheckForUpdates/CheckForUpdates.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 952a2dfbc..d6068148e 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -26,6 +26,7 @@ DownloadAndImportBcContainerHelper if ($token) { # Specified token is GhTokenWorkflow secret - decode from base 64 + Write-Host "Using ghTokenWorkflow secret" $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } else { @@ -33,6 +34,7 @@ else { throw "A personal access token with permissions to modify Workflows is needed. You must add a secret called GhTokenWorkflow containing a personal access token. You can Generate a new token from https://github.com/settings/tokens. Make sure that the workflow scope is checked." } else { + Write-Host "Using GitHub Token" $token = $githubToken } } From 26149ded3e120e4f3b95e460fec1b8e92d8a3d17 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 22 Aug 2024 13:00:29 +0200 Subject: [PATCH 094/108] check if permissions --- Actions/CheckForUpdates/yamlclass.ps1 | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index d8ae1016d..a109ed574 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -443,10 +443,14 @@ class Yaml { } # Merge permissions Write-host "Merge permissions" - $srcPermissions = [Yaml]::GetPermissionsFromArray($srcYaml.Get('permissions:/').content) - $yamlPermissions = [Yaml]::GetPermissionsFromArray($yaml.Get('permissions:/').content) - if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { - $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + $srcPermissionsObj = $srcYaml.Get('permissions:/') + $yamlPermissionsObj = $yaml.Get('permissions:/') + if ($srcPermissionsObj -and $yamlPermissionsObj) { + $srcPermissions = [Yaml]::GetPermissionsFromArray($srcPermissionsObj.content) + $yamlPermissions = [Yaml]::GetPermissionsFromArray($yamlPermissionsObj.content) + if ("$srcPermissions" -ne "" -and "$yamlPermissions" -ne "") { + $srcYaml.Replace('permissions:/', [Yaml]::GetPermissionsArray([Yaml]::MergePermissions($srcPermissions, $yamlPermissions))) + } } # Apply cystom steps From 0e6e7c31a1b16cf3587e35dbb3f83efa121dc834 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 2 Sep 2024 08:30:37 +0200 Subject: [PATCH 095/108] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index f6095e957..a1946081c 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -2,7 +2,7 @@ AL-Go for GitHub is a plug-and-play DevOps solution, intended to support 100% of the functionality needed by 90% of the people developing applications for Microsoft Dynamics 365 Business Central out-of-the-box. -If AL-Go functionality out-of-the-box doesn't match your needs, you really three options: +If AL-Go functionality out-of-the-box doesn't match your needs, you really have three options: 1. Customize AL-Go for GitHub to fit your needs 1. Select another managed DevOps solution From 858865e95f94ba930813d7ea845d256bdd8c35f7 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 2 Sep 2024 08:30:52 +0200 Subject: [PATCH 096/108] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index a1946081c..6403cbe9b 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -23,7 +23,7 @@ There are three ways you can customize AL-Go for GitHub to fit your needs. You c 1. customize the repository with custom scripts, workflows, jobs or steps following the guidelines below 1. create a customized repository and use this as your template repository (indirect template) -1. fork the AL-Go for GitHub and create your "own" version +1. fork the AL-Go for GitHub and create your "own" version (not recommended) > \[!CAUTION\] > The more you customize AL-Go for GitHub, the more likely you are to be broken by future updates to AL-Go for GitHub, meaning that you will have to update your customizations to match the changes in AL-Go for GitHub. From f252f296cd6b19946ca9b245a867422d3a3bda74 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 3 Sep 2024 14:25:13 +0200 Subject: [PATCH 097/108] empty line --- RELEASENOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index abd73e308..e7ec62523 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,4 +1,5 @@ ### Issues + - Issue 1184 Publish to Environment fails on 'Permission Denied' ### New Settings From 2e7722ad04af521d257d172890f44ac730b976c2 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 9 Sep 2024 14:14:29 +0200 Subject: [PATCH 098/108] Update Scenarios/CustomizingALGoForGitHub.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/CustomizingALGoForGitHub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md index 6403cbe9b..5dfad600c 100644 --- a/Scenarios/CustomizingALGoForGitHub.md +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -172,7 +172,7 @@ By adding a setting called [`customALGoSystemFiles`](https://aka.ms/algosettings | :-- | :-- | :-: | :-- | | Destination | Path in which the file should be placed. Can include the filename if the source doesn't point to a .zip file, must include a terminating / or \\ if a filename is not included. | Yes | | | Source | URL to a either a single file or a .zip file containing custom AL-Go System Files. Must be https. | Yes | | -| FileSpec | If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | +| FileSpec | If the source URL points to a .zip file, this property can specify which files to include. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. | No | * | | Recurse | Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) | No | true | This setting will cause AL-Go for GitHub to include these files during the next update. From a76dc7deaff8862038506c76d8174215144080a1 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 9 Sep 2024 14:14:46 +0200 Subject: [PATCH 099/108] Update Scenarios/settings.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 524688c6e..8a582b9af 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -187,7 +187,7 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza # Expert level -The settings and functionality in the expert section might requires knowledge about GitHub Workflows/Actions, yaml, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. +The settings and functionality in the expert section might require knowledge about GitHub Workflows/Actions, YAML, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. Please read the release notes carefully when installing new versions of AL-Go for GitHub. From ddef716f38001eee63b1457f711c4684eac3dbd2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 13 Sep 2024 12:49:48 -0400 Subject: [PATCH 100/108] prop --- Actions/AL-Go-Helper.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 65335006c..e85599238 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -465,6 +465,10 @@ function MergeCustomObjectIntoOrderedDictionary { if ($srcPropType -eq "PSCustomObject" -and $dstPropType -eq "OrderedDictionary") { MergeCustomObjectIntoOrderedDictionary -dst $dst."$prop" -src $srcProp } + elseif ($dstProp -is [String] -and $srcProp -is [Object[]]) { + # For properties like "runs-on", which is a string, you can specify an array in settings, which gets joined with a comma + $dst."$prop" = $srcProp -join ', ' + } elseif ($dstPropType -ne $srcPropType -and !($srcPropType -eq "Int64" -and $dstPropType -eq "Int32")) { # Under Linux, the Int fields read from the .json file will be Int64, while the settings defaults will be Int32 # This is not seen as an error and will not throw an error From 2ca9c43b14a14316b7cddfcb3775b421140ebf64 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 13 Oct 2024 08:25:51 +0200 Subject: [PATCH 101/108] merge --- .../AppSource App/.github/workflows/_BuildALGoProject.yaml | 2 +- .../.github/workflows/_BuildALGoProject.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index ba5187c9e..c5ba272db 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -106,7 +106,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS,cicdAuthContext' + getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,cicdAuthContext' - name: Determine ArtifactUrl uses: microsoft/AL-Go-Actions/DetermineArtifactUrl@main diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index ba5187c9e..c5ba272db 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -106,7 +106,7 @@ jobs: with: shell: ${{ inputs.shell }} gitHubSecrets: ${{ toJson(secrets) }} - getSecrets: '${{ inputs.secrets }},appDependencyProbingPathsSecrets,AZURE_CREDENTIALS,cicdAuthContext' + getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,cicdAuthContext' - name: Determine ArtifactUrl uses: microsoft/AL-Go-Actions/DetermineArtifactUrl@main From dc4580f19b9668ea5f9dc05df6917531c07e762b Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 13 Oct 2024 08:27:18 +0200 Subject: [PATCH 102/108] merge --- .github/workflows/jekyll-gh-pages.yml | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml new file mode 100644 index 000000000..10ed10e90 --- /dev/null +++ b/.github/workflows/jekyll-gh-pages.yml @@ -0,0 +1,49 @@ +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Deploy Jekyll with GitHub Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Setup Pages + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + - name: Build with Jekyll + uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 + with: + source: . + destination: ./_site + - name: Upload artifact + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 + with: + preview: true From 2cf539e33eb1eb58177b474f1b3efcf5b3000a53 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 29 Oct 2024 07:50:50 +0100 Subject: [PATCH 103/108] remove --- .github/workflows/jekyll-gh-pages.yml | 49 --------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/jekyll-gh-pages.yml diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml deleted file mode 100644 index 10ed10e90..000000000 --- a/.github/workflows/jekyll-gh-pages.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Sample workflow for building and deploying a Jekyll site to GitHub Pages -name: Deploy Jekyll with GitHub Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - name: Setup Pages - uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - - name: Build with Jekyll - uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 - with: - source: . - destination: ./_site - - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 - with: - preview: true From 3ad4cbe6bdd0a5575a3de1040985afc53700048b Mon Sep 17 00:00:00 2001 From: Patrick Schiefer <35697862+PatrickSchiefer@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:59:48 +0100 Subject: [PATCH 104/108] feat: add needsContext as env to build process (#206) --- Templates/AppSource App/.github/workflows/CICD.yaml | 1 + .../AppSource App/.github/workflows/_BuildALGoProject.yaml | 5 +++++ Templates/Per Tenant Extension/.github/workflows/CICD.yaml | 1 + .../.github/workflows/_BuildALGoProject.yaml | 5 +++++ 4 files changed, 12 insertions(+) diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index 74fc6fac8..796d68a29 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -172,6 +172,7 @@ jobs: publishArtifacts: ${{ github.ref_name == 'main' || startswith(github.ref_name, 'release/') || startswith(github.ref_name, 'releases/') || needs.Initialization.outputs.deliveryTargetsJson != '[]' || needs.Initialization.outputs.environmentCount > 0 }} signArtifacts: true useArtifactCache: true + needsContext: ${{ toJson(needs) }} DeployALDoc: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index a8f4017fd..91b3fc308 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -67,6 +67,10 @@ on: description: Flag determining whether to use the Artifacts Cache type: boolean default: false + needsContext: + description: JSON formated needs context + type: string + default: false permissions: actions: read @@ -139,6 +143,7 @@ jobs: env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} + NeedsContext: ${{ inputs.needsContext }} with: shell: ${{ inputs.shell }} artifact: ${{ env.artifact }} diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 307dca38e..a7068a0dd 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -172,6 +172,7 @@ jobs: publishArtifacts: ${{ github.ref_name == 'main' || startswith(github.ref_name, 'release/') || startswith(github.ref_name, 'releases/') || needs.Initialization.outputs.deliveryTargetsJson != '[]' || needs.Initialization.outputs.environmentCount > 0 }} signArtifacts: true useArtifactCache: true + needsContext: ${{ toJson(needs) }} BuildPP: needs: [ Initialization ] diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index a8f4017fd..91b3fc308 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -67,6 +67,10 @@ on: description: Flag determining whether to use the Artifacts Cache type: boolean default: false + needsContext: + description: JSON formated needs context + type: string + default: false permissions: actions: read @@ -139,6 +143,7 @@ jobs: env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} + NeedsContext: ${{ inputs.needsContext }} with: shell: ${{ inputs.shell }} artifact: ${{ env.artifact }} From 1355bcaf593cb93b354ad0ab34e8662b64bd2080 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 29 Oct 2024 09:02:04 +0100 Subject: [PATCH 105/108] modify default --- .../AppSource App/.github/workflows/_BuildALGoProject.yaml | 4 ++-- .../.github/workflows/_BuildALGoProject.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index 91b3fc308..b2bf7ab39 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -68,9 +68,9 @@ on: type: boolean default: false needsContext: - description: JSON formated needs context + description: JSON formatted needs context type: string - default: false + default: '{}' permissions: actions: read diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index 91b3fc308..b2bf7ab39 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -68,9 +68,9 @@ on: type: boolean default: false needsContext: - description: JSON formated needs context + description: JSON formatted needs context type: string - default: false + default: '{}' permissions: actions: read From 3fa79d6ce8d58947cb3213c0e22e7fccbab3abdb Mon Sep 17 00:00:00 2001 From: Patrick Schiefer <35697862+PatrickSchiefer@users.noreply.github.com> Date: Tue, 29 Oct 2024 20:17:38 +0100 Subject: [PATCH 106/108] feat: add needContext to Build job (#208) --- Templates/AppSource App/.github/workflows/Current.yaml | 1 + Templates/AppSource App/.github/workflows/NextMajor.yaml | 1 + Templates/AppSource App/.github/workflows/NextMinor.yaml | 1 + .../AppSource App/.github/workflows/PullRequestHandler.yaml | 1 + Templates/Per Tenant Extension/.github/workflows/Current.yaml | 1 + Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml | 1 + Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml | 1 + .../.github/workflows/PullRequestHandler.yaml | 1 + 8 files changed, 8 insertions(+) diff --git a/Templates/AppSource App/.github/workflows/Current.yaml b/Templates/AppSource App/.github/workflows/Current.yaml index 9226401b1..1d8c4ee26 100644 --- a/Templates/AppSource App/.github/workflows/Current.yaml +++ b/Templates/AppSource App/.github/workflows/Current.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'Current' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/NextMajor.yaml b/Templates/AppSource App/.github/workflows/NextMajor.yaml index ec3bade7f..0d0f77b53 100644 --- a/Templates/AppSource App/.github/workflows/NextMajor.yaml +++ b/Templates/AppSource App/.github/workflows/NextMajor.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMajor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/NextMinor.yaml b/Templates/AppSource App/.github/workflows/NextMinor.yaml index 79ad8e15e..4dff645ca 100644 --- a/Templates/AppSource App/.github/workflows/NextMinor.yaml +++ b/Templates/AppSource App/.github/workflows/NextMinor.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMinor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml b/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml index 2d452db89..15482ee49 100644 --- a/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml +++ b/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml @@ -99,6 +99,7 @@ jobs: secrets: 'licenseFileUrl,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'PR${{ github.event.number }}' + needsContext: ${{ toJson(needs) }} StatusCheck: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/Current.yaml b/Templates/Per Tenant Extension/.github/workflows/Current.yaml index 9226401b1..1d8c4ee26 100644 --- a/Templates/Per Tenant Extension/.github/workflows/Current.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/Current.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'Current' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml b/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml index ec3bade7f..0d0f77b53 100644 --- a/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMajor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml b/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml index 79ad8e15e..4dff645ca 100644 --- a/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml @@ -82,6 +82,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'NextMinor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml b/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml index 2d452db89..15482ee49 100644 --- a/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml @@ -99,6 +99,7 @@ jobs: secrets: 'licenseFileUrl,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' publishThisBuildArtifacts: ${{ needs.Initialization.outputs.workflowDepth > 1 }} artifactsNameSuffix: 'PR${{ github.event.number }}' + needsContext: ${{ toJson(needs) }} StatusCheck: needs: [ Initialization, Build ] From bbd9cd2121f7771656d0b85d851cd9079717f905 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 18 Dec 2024 23:01:07 +0100 Subject: [PATCH 107/108] move --- RELEASENOTES.md | 54 ++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1cd003ba7..56da4fd55 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -2,6 +2,33 @@ - It is now possible to skip the modification of dependency version numbers when running the Increment Version number workflow or the Create Release workflow +### New Settings + +- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: + + - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) + - **Source** (mandatory) - URL to a either a single file or a .zip file containing + - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. + - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) + +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Add custom steps to the AL-Go build workflow + +It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. + +Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Indirect AL-Go template repository + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. + +Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. + ### New Versioning Strategy Setting versioning strategy to 3 will allow 3 segments of the version number to be defined in app.json and repoVersion. Only the 4th segment (Revision) will be defined by the GitHub [run_number](https://go.microsoft.com/fwlink/?linkid=2217416&clcid=0x409) for the CI/CD workflow. Increment version number and Create Release now also supports the ability to set a third segment to the RepoVersion and appversion in app.json. @@ -82,33 +109,6 @@ Page Scripting tests are now supported as part of CI/CD. By specifying pageScrip With this release the implementation for delivering to NuGet packages (by adding the NuGetContext secret), is similar to the functionality behind delivering to GitHub packages and the implementation is no longer in preview. -### New Settings - -- `customALGoSystemFiles` is an array of JSON objects, which holds information about custom AL-Go System Files, which will be applied during Update AL-Go System Files. Every object can hold these 4 properties: - - - **Destination** (mandatory) - Path in which the file should be placed (Can include the filename if the source doesn't point to a .zip file) - - **Source** (mandatory) - URL to a either a single file or a .zip file containing - - **FileSpec** (optional) - If the source URL points to a .zip file, this property can specify which files to include if the source URL points to a .zip file. The FileSpec can include a subfolder inside the .zip file, and must include a file name pattern. - - **Recurse** (optional) - Include all files matching the file name pattern in FileSpec from all subfolders (under a given subfolder from FileSpec) - -### Add custom jobs to AL-Go workflows - -It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. - -Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Add custom steps to the AL-Go build workflow - -It is now possible to add custom steps to the AL-Go for GitHub `BuildALGoProject` job in the `_BuildALGoProject.yaml` file. Custom steps needs to be named `CustomStep` and can be placed at one of a set of predefined anchors. See [https://aka.ms/algosettings#customsteps](https://aka.ms/algosettings#customsteps) for details. - -Note that custom steps might break by future changes to AL-Go for GitHub workflows. If you have customized steps to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. - -### Indirect AL-Go template repository - -Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs, custom steps and/or settings to this repository and then use that repository as the template repository for other repositories. Using indirect template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#indirect](https://aka.ms/algosettings#indirect) for details. - -Note that customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always doublecheck the pull request when updating AL-Go system files in your indirect template repositories. - ### Allow GitHubRunner and GitHubRunnerShell as project settings Previously, AL-Go required the GitHubRunner and GitHubRunnerShell settings to be set on repository level. This has now been changed such that they can be set on project level. From 11a3f6785af383b32691259b3793241a5302bbaa Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 25 Dec 2024 12:27:12 -0800 Subject: [PATCH 108/108] or --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 32a298a2b..245a006ab 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -566,7 +566,7 @@ function GetCustomALGoSystemFiles { if ($fileSpec -isnot [string] -or $recurse -isnot [boolean]) { throw "customALGoSystemFiles setting is wrongly formatted, fileSpec must be string and Recurse must be boolean. See https://aka.ms/algosettings#customalgosystemfiles." } - if (!($destination.EndsWith([IO.Path]::DirectorySeparatorChar))) { + if (!($destination.EndsWith('/') -or $destination.EndsWith('\'))) { throw "customALGoSystemFiles setting is wrongly formatted, destination must be a folder (terminated with / or \). See https://aka.ms/algosettings#customalgosystemfiles." } Invoke-RestMethod -UseBasicParsing -Method Get -Uri $source -OutFile $zipName