From c47d8cafd9b887cb84c9bbf99f0ede7a6220a1b2 Mon Sep 17 00:00:00 2001 From: Markus Lippert Date: Fri, 15 Nov 2024 13:07:52 +0100 Subject: [PATCH] [BcNuGet] Reduce memory consumption, improve performance, prevent memory leaks (#3760) We found that `Compress-Archive` and especially storing the `octet-stream` inside a string instead of using a Stream causes a very high memory consumption and a bad performance. We're invoking the BcNuGet Cmdlets from our own C# API and therefore were able to profile the memory and performance. The behaviour however is similar when invoking the Cmdlets directly (PowerShell Core 7.4.6). **Memory (Note: around 340 MB is the base load our API brings with it)** Before: Memory spikes when uploading big apps, around 1 GB memory consumption from BcNuGet Cmdlets image After: image **Performance** Before: image After: image --- NuGet/New-BcNuGetPackage.ps1 | 6 +++++- NuGet/NuGetFeedClass.ps1 | 20 +++++++++++--------- ReleaseNotes.txt | 1 + 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/NuGet/New-BcNuGetPackage.ps1 b/NuGet/New-BcNuGetPackage.ps1 index 9861390f6..de20ab580 100644 --- a/NuGet/New-BcNuGetPackage.ps1 +++ b/NuGet/New-BcNuGetPackage.ps1 @@ -286,7 +286,11 @@ Function New-BcNuGetPackage { if (Test-Path $nuPkgFile -PathType Leaf) { Remove-Item $nupkgFile -Force } - Compress-Archive -Path "$rootFolder\*" -DestinationPath "$nupkgFile.zip" -Force + if (Test-Path "$nupkgFile.zip" -PathType Leaf) { + Remove-Item "$nupkgFile.zip" -Force + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::CreateFromDirectory("$rootFolder", "$nupkgFile.zip", [System.IO.Compression.CompressionLevel]::Optimal, $false) Rename-Item -Path "$nupkgFile.zip" -NewName $nuPkgFileName $size = (Get-Item $nupkgFile).Length diff --git a/NuGet/NuGetFeedClass.ps1 b/NuGet/NuGetFeedClass.ps1 index b9d669dfa..612ae9878 100644 --- a/NuGet/NuGetFeedClass.ps1 +++ b/NuGet/NuGetFeedClass.ps1 @@ -353,17 +353,19 @@ class NuGetFeed { "X-NuGet-ApiKey" = $this.token "X-NuGet-Client-Version" = "6.3.0" } - $FileContent = [System.IO.File]::ReadAllBytes($package) - $boundary = [System.Guid]::NewGuid().ToString(); + $boundary = [System.Guid]::NewGuid().ToString(); $LF = "`r`n"; - - $body = [System.Text.Encoding]::UTF8.GetBytes("--$boundary$LF") - $body += [System.Text.Encoding]::UTF8.GetBytes("Content-Type: application/octet-stream$($LF)Content-Disposition: form-data; name=package; filename=""$([System.IO.Path]::GetFileName($package))""$($LF)$($LF)") - $body += $fileContent - $body += [System.Text.Encoding]::UTF8.GetBytes("$LF--$boundary--$LF") - $tmpFile = Join-Path ([System.IO.Path]::GetTempPath()) ([GUID]::NewGuid().ToString()) - [System.IO.File]::WriteAllBytes($tmpFile, $body) + $fs = [System.IO.File]::OpenWrite($tmpFile) + try { + $fs.Write([System.Text.Encoding]::UTF8.GetBytes("--$boundary$LF")) + $fs.Write([System.Text.Encoding]::UTF8.GetBytes("Content-Type: application/octet-stream$($LF)Content-Disposition: form-data; name=package; filename=""$([System.IO.Path]::GetFileName($package))""$($LF)$($LF)")) + $fs.Write([System.IO.File]::ReadAllBytes($package)) + $fs.Write([System.Text.Encoding]::UTF8.GetBytes("$LF--$boundary--$LF")) + } finally { + $fs.Close() + } + Write-Host "Submitting NuGet package" try { Invoke-RestMethod -UseBasicParsing -Uri $this.packagePublishUrl -ContentType "multipart/form-data; boundary=$boundary" -Method Put -Headers $headers -inFile $tmpFile | Out-Host diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 4bc17f71a..5d6158178 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -1,5 +1,6 @@ 6.0.29 Issue 3591 When using Publish-NAVApp to publish an app, which fails compilation in the service, the command might hang forever - the fix for this is a temporary hack put in place for the versions which doesn't work. +Improve performance and reduce memory consumption when creating and pushing NuGet packages 6.0.28 Set useApproximateVersion to true in settings to increase performance of Get-BcArtifactUrl when selecting latest artifact, by using an approximate filtering on blobs