From dec1dc784c1dbc2c2712197f2d2ed8b5be7c3e67 Mon Sep 17 00:00:00 2001 From: Maxwell Weru Date: Sun, 30 Jul 2023 16:12:06 +0300 Subject: [PATCH] Image per ecosystem (#720) This PR moves from the single image which is now dated, on to having an image per ecosystem. This alignes with the GitHub hosted version. Some more information available [here](https://github.com/dependabot/dependabot-script/issues/896#issuecomment-1425382734) As a consequence, the image repository can no longer be specified as an input to the task (using `dockerImageRepository`) or the server (using `updaterImageRepository`). --- .github/workflows/cleanup.yml | 32 +++++++++++++++++-- .github/workflows/updater.yml | 29 +++++++++++++++-- cronjob-template.yaml | 2 +- docs/server.md | 1 - docs/updater.md | 8 ++--- extension/task/index.ts | 6 ++-- extension/task/task.json | 9 ------ extension/task/utils/getSharedVariables.ts | 7 ---- .../Workflow/UpdateRunnerTests.cs | 10 +++--- .../Workflow/UpdateRunner.cs | 5 +-- .../Workflow/WorkflowConfigureOptions.cs | 4 +-- .../Workflow/WorkflowOptions.cs | 6 ++-- server/Tingle.Dependabot/appsettings.json | 2 +- server/main.bicep | 5 +-- server/main.json | 11 ++----- updater/Dockerfile | 20 +++--------- 16 files changed, 85 insertions(+), 72 deletions(-) diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 638a1807..71e39c69 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -6,18 +6,44 @@ on: - cron: '0 0 */14 * *' # every 14 days jobs: - cleanup: + cleanup-updater: runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + suite: + - { name: bundler } + - { name: cargo } + - { name: composer } + - { name: docker } + - { name: elm } + - { name: gitsubmodule } + - { name: github-actions } + - { name: gomod } + - { name: gradle } + - { name: mix } + - { name: maven } + - { name: npm } + - { name: nuget } + - { name: pub } + - { name: pip } + # TODO: restore swift here once supported elsewhere + # - { name: swift } + - { name: terraform } steps: - - name: Delete old dependabot-updater images + - name: Delete old dependabot-updater-${{ matrix.suite.ecosystem }} images uses: actions/delete-package-versions@v4 with: - package-name: 'dependabot-updater' + package-name: 'dependabot-updater-${{ matrix.suite.ecosystem }}' package-type: 'container' min-versions-to-keep: 20 delete-only-pre-release-versions: "true" + cleanup-server: + runs-on: ubuntu-latest + + steps: - name: Delete old dependabot-server images uses: actions/delete-package-versions@v4 with: diff --git a/.github/workflows/updater.yml b/.github/workflows/updater.yml index ec89b4a4..e029da75 100644 --- a/.github/workflows/updater.yml +++ b/.github/workflows/updater.yml @@ -21,9 +21,31 @@ on: jobs: Build: runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + suite: + - { name: bundler, ecosystem: bundler } + - { name: cargo, ecosystem: cargo } + - { name: composer, ecosystem: composer } + - { name: docker, ecosystem: docker } + - { name: elm, ecosystem: elm } + - { name: git_submodules, ecosystem: gitsubmodule } + - { name: github_actions, ecosystem: github-actions } + - { name: go_modules, ecosystem: gomod } + - { name: gradle, ecosystem: gradle } + - { name: hex, ecosystem: mix } + - { name: maven, ecosystem: maven } + - { name: npm_and_yarn, ecosystem: npm } + - { name: nuget, ecosystem: nuget } + - { name: pub, ecosystem: pub } + - { name: python, ecosystem: pip } + # TODO: restore swift here once supported elsewhere + # - { name: swift, ecosystem: swift } + - { name: terraform, ecosystem: terraform } env: - IMAGE_NAME: 'dependabot-updater' + IMAGE_NAME: 'dependabot-updater-${{ matrix.suite.ecosystem }}' DOCKER_BUILDKIT: 1 # Enable Docker BuildKit # $BUNDLE_GEMFILE must be set at the job level, so it is set for all steps BUNDLE_GEMFILE: ${{ github.workspace }}/updater/Gemfile @@ -59,11 +81,15 @@ jobs: - name: Pull Docker base image & warm Docker cache run: docker pull "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:latest" + # TODO: remove this after at least one release tagged 'latest' + continue-on-error: true - name: Build image run: | docker build \ -f updater/Dockerfile \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --build-arg ECOSYSTEM=${{ matrix.suite.ecosystem }} \ --label com.github.image.run.id=$GITHUB_RUN_ID \ --label com.github.image.run.number=$GITHUB_RUN_NUMBER \ --label com.github.image.job.id=$GITHUB_JOB \ @@ -75,7 +101,6 @@ jobs: -t "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:$GITVERSION_MAJOR.$GITVERSION_MINOR" \ -t "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:$GITVERSION_MAJOR" \ --cache-from ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:latest \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ . - name: Log into registry diff --git a/cronjob-template.yaml b/cronjob-template.yaml index 92f373a1..aef68d2c 100644 --- a/cronjob-template.yaml +++ b/cronjob-template.yaml @@ -36,7 +36,7 @@ spec: activeDeadlineSeconds: 3600 containers: - name: dependabot - image: 'ghcr.io/tinglesoftware/dependabot-updater' # Use a specific tag otherwise the image may be cached already + image: 'ghcr.io/tinglesoftware/dependabot-updater-{{dependabot_package_ecosystem}}' # Use a specific tag otherwise the image may be cached already resources: {} # decide based on your usage env: - name: GITHUB_ACCESS_TOKEN diff --git a/docs/server.md b/docs/server.md index de8dd183..3ec05bd7 100644 --- a/docs/server.md +++ b/docs/server.md @@ -69,7 +69,6 @@ The deployment exposes the following parameters that can be tuned to suit the se |dockerImageRegistry|The docker registry to use when pulling the docker containers if needed. By default this will GitHub Container Registry. This can be useful if the container needs to come from an internal docker registry mirror or alternative source for testing. If the registry requires authentication ensure to assign `acrPull` permissions to the managed identity.
Example: `contoso.azurecr.io`|No|`ghcr.io`| |serverImageRepository|The docker container repository to use when pulling the server docker container. This can be useful if the default container requires customizations such as custom certs.|No|`tinglesoftware/dependabot-server`| |serverImageTag|The image tag to use when pulling the docker container. A tag also defines the version. You should avoid using `latest`. Example: `0.1.0`|No|<version-downloaded>| -|updaterImageRepository|The docker container repository to use when pulling the updater docker container. This can be useful if the default container requires customizations such as custom certs.|No|`tinglesoftware/dependabot-updater`| |updaterImageTag|The image tag to use when pulling the updater docker container. A tag also defines the version. You should avoid using `latest`. Example: `0.1.0`|No|<version-downloaded>| |minReplicas|The minimum number of replicas to required for the deployment. Given that scheduling runs in process, this value cannot be less than `1`. This may change in the future.|No|1| |maxReplicas|The maximum number of replicas when automatic scaling engages. In most cases, you do not need more than 1.|No|1| diff --git a/docs/updater.md b/docs/updater.md index 91c9342f..d838ad1e 100644 --- a/docs/updater.md +++ b/docs/updater.md @@ -3,7 +3,7 @@ First, you need to pull the docker image locally to your machine: ```bash -docker pull ghcr.io/tinglesoftware/dependabot-updater +docker pull ghcr.io/tinglesoftware/dependabot-updater- ``` Next create and run a container from the image: @@ -32,7 +32,7 @@ docker run --rm -t \ -e AZURE_SET_AUTO_COMPLETE= \ -e AZURE_AUTO_APPROVE_PR= \ -e AZURE_AUTO_APPROVE_USER_TOKEN= \ - ghcr.io/tinglesoftware/dependabot-updater + ghcr.io/tinglesoftware/dependabot-updater- ``` An example, for Azure DevOps Services: @@ -58,7 +58,7 @@ docker run --rm -t \ -e AZURE_SET_AUTO_COMPLETE=true \ -e AZURE_AUTO_APPROVE_PR=true \ -e AZURE_AUTO_APPROVE_USER_TOKEN=ijkl..mnop \ - ghcr.io/tinglesoftware/dependabot-updater + ghcr.io/tinglesoftware/dependabot-updater-nuget ``` An example, for Azure DevOps Server: @@ -87,7 +87,7 @@ docker run --rm -t \ -e AZURE_SET_AUTO_COMPLETE=true \ -e AZURE_AUTO_APPROVE_PR=true \ -e AZURE_AUTO_APPROVE_USER_TOKEN=ijkl..mnop \ - ghcr.io/tinglesoftware/dependabot-updater + ghcr.io/tinglesoftware/dependabot-updater-nuget ``` ## Environment Variables diff --git a/extension/task/index.ts b/extension/task/index.ts index 2c70798a..4735d741 100644 --- a/extension/task/index.ts +++ b/extension/task/index.ts @@ -221,9 +221,9 @@ async function run() { dockerRunner.arg(['--mount', `type=bind,source=/ssh-agent,target=/ssh-agent`]); } - // Form the docker image based on the repository and the tag, e.g. tinglesoftware/dependabot-updater - // For custom/enterprise registries, prefix with the registry, e.g. contoso.azurecr.io/tinglesoftware/dependabot-updater - let dockerImage: string = `${variables.dockerImageRepository}:${variables.dockerImageTag}`; + // Form the docker image based on the ecosystem (repository) and the tag e.g. tinglesoftware/dependabot-updater-nuget + // For custom/enterprise registries, prefix with the registry, e.g. contoso.azurecr.io/tinglesoftware/dependabot-updater-nuget + let dockerImage: string = `tinglesoftware/dependabot-updater-${update.packageEcosystem}:${variables.dockerImageTag}` if (variables.dockerImageRegistry) { dockerImage = `${variables.dockerImageRegistry}/${dockerImage}`.replace("//", "/"); } diff --git a/extension/task/task.json b/extension/task/task.json index 74f2906b..ce5c4c42 100644 --- a/extension/task/task.json +++ b/extension/task/task.json @@ -216,15 +216,6 @@ "defaultValue": "ghcr.io", "required": false }, - { - "name": "dockerImageRepository", - "label": "Docker Container repository", - "type": "string", - "helpMarkDown": "The docker container repository to use when pulling the docker container used by the task if needed. This can be useful if the default container requires customizations such as custom certs.", - "defaultValue": "tinglesoftware/dependabot-updater", - "required": true, - "groupName": "advanced" - }, { "name": "dockerImageTag", "type": "string", diff --git a/extension/task/utils/getSharedVariables.ts b/extension/task/utils/getSharedVariables.ts index 74ff31f8..8049b8c1 100644 --- a/extension/task/utils/getSharedVariables.ts +++ b/extension/task/utils/getSharedVariables.ts @@ -66,8 +66,6 @@ export interface ISharedVariables { /** Registry of the docker image to be pulled */ dockerImageRegistry: string | undefined; - /** Repository of the docker image to be pulled */ - dockerImageRepository: string; /** Tag of the docker image to be pulled */ dockerImageTag: string; } @@ -148,10 +146,6 @@ export default function getSharedVariables(): ISharedVariables { let dockerImageRegistry: string | undefined = tl.getInput( "dockerImageRegistry" ); - let dockerImageRepository: string = tl.getInput( - "dockerImageRepository", - true - ); let dockerImageTag: string = getDockerImageTag(); return { @@ -189,7 +183,6 @@ export default function getSharedVariables(): ISharedVariables { forwardHostSshSocket, dockerImageRegistry, - dockerImageRepository, dockerImageTag, }; } diff --git a/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs b/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs index ea5d0719..a2018213 100644 --- a/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs +++ b/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs @@ -222,11 +222,11 @@ public void ConvertPlaceholder_Works() } [Theory] - [InlineData("contoso.azurecr.io/tinglesoftware/dependabot-updater:0.11", true, "contoso.azurecr.io")] - [InlineData("fabrikam.azurecr.io/tinglesoftware/dependabot-updater:0.11", true, "fabrikam.azurecr.io")] - [InlineData("dependabot.azurecr.io/tinglesoftware/dependabot-updater:0.11", true, "dependabot.azurecr.io")] - [InlineData("ghcr.io/tinglesoftware/dependabot-updater:0.11", false, null)] - [InlineData("tingle/dependabot-updater:0.11", false, null)] + [InlineData("contoso.azurecr.io/tinglesoftware/dependabot-updater-nuget:0.11", true, "contoso.azurecr.io")] + [InlineData("fabrikam.azurecr.io/tinglesoftware/dependabot-updater-nuget:0.11", true, "fabrikam.azurecr.io")] + [InlineData("dependabot.azurecr.io/tinglesoftware/dependabot-updater-nuget:0.11", true, "dependabot.azurecr.io")] + [InlineData("ghcr.io/tinglesoftware/dependabot-updater-nuget:0.11", false, null)] + [InlineData("tingle/dependabot-updater-nuget:0.11", false, null)] [InlineData("tingle/dependabot-azure-devops:0.11", false, null)] public void TryGetAzureContainerRegistry_Works(string input, bool matches, string? expected) { diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index 2f7d44d0..e733fb1c 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -57,7 +57,8 @@ public async Task CreateAsync(Repository repository, RepositoryUpdate update, Up catch (Azure.RequestFailedException rfe) when (rfe.Status is 404) { } // prepare the container - var container = new ContainerInstanceContainer(UpdaterContainerName, options.UpdaterContainerImage, new(job.Resources!)); + var image = options.UpdaterContainerImageTemplate!.Replace("{{ecosystem}}", job.PackageEcosystem.GetEnumMemberAttrValueOrDefault()); + var container = new ContainerInstanceContainer(UpdaterContainerName, image, new(job.Resources!)); var env = CreateVariables(repository, update, job); foreach (var (key, value) in env) container.EnvironmentVariables.Add(new ContainerEnvironmentVariable(key) { Value = value, }); @@ -73,7 +74,7 @@ public async Task CreateAsync(Repository repository, RepositoryUpdate update, Up }; // add credentials for pulling image(s) from azure container registry - if (TryGetAzureContainerRegistry(options.UpdaterContainerImage!, out var registry)) + if (TryGetAzureContainerRegistry(image, out var registry)) { data.ImageRegistryCredentials.Add(new ContainerGroupImageRegistryCredential(registry) { Identity = options.ManagedIdentityId, }); } diff --git a/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs index 8d81f394..a463860c 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowConfigureOptions.cs @@ -53,9 +53,9 @@ public ValidateOptionsResult Validate(string? name, WorkflowOptions options) return ValidateOptionsResult.Fail($"'{nameof(options.LogAnalyticsWorkspaceKey)}' cannot be null or whitespace"); } - if (string.IsNullOrWhiteSpace(options.UpdaterContainerImage)) + if (string.IsNullOrWhiteSpace(options.UpdaterContainerImageTemplate)) { - return ValidateOptionsResult.Fail($"'{nameof(options.UpdaterContainerImage)}' cannot be null or whitespace"); + return ValidateOptionsResult.Fail($"'{nameof(options.UpdaterContainerImageTemplate)}' cannot be null or whitespace"); } if (string.IsNullOrWhiteSpace(options.ManagedIdentityId)) diff --git a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs index b59c4924..5bde28da 100644 --- a/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs +++ b/server/Tingle.Dependabot/Workflow/WorkflowOptions.cs @@ -36,14 +36,14 @@ public class WorkflowOptions public string? ManagedIdentityId { get; set; } /// - /// The registry, repository and version of the docker container image to use. + /// Template representing the docker container image to use. /// Keeping this value fixed in code is important so that the code that depends on it always works. /// More like a dependency. ///
/// However, in production there maybe an issue that requires a rollback hence the value is placed in options. /// - /// ghcr.io/tinglesoftware/dependabot-updater:0.14 - public string? UpdaterContainerImage { get; set; } + /// ghcr.io/tinglesoftware/dependabot-updater-{{ecosystem}}:0.14 + public string? UpdaterContainerImageTemplate { get; set; } /// URL for the project. public AzureDevOpsProjectUrl? ProjectUrl { get; set; } diff --git a/server/Tingle.Dependabot/appsettings.json b/server/Tingle.Dependabot/appsettings.json index 8c6a729b..ce551066 100644 --- a/server/Tingle.Dependabot/appsettings.json +++ b/server/Tingle.Dependabot/appsettings.json @@ -64,7 +64,7 @@ "LogAnalyticsWorkspaceId": "00000000-0000-1111-0001-000000000000", "LogAnalyticsWorkspaceKey": "AAAAAAAAAAA=", "ManagedIdentityId": "/subscriptions/00000000-0000-1111-0001-000000000000/resourceGroups/DEPENDABOT/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dependabot", - "UpdaterContainerImage": "ghcr.io/tinglesoftware/dependabot-updater:0.14.2-ci.37", + "UpdaterContainerImageTemplate": "ghcr.io/tinglesoftware/dependabot-updater-{{ecosystem}}:0.14.2-ci.37", "ProjectUrl": "https://dev.azure.com/fabrikam/DefaultCollection", "ProjectToken": "", "GithubToken": "", diff --git a/server/main.bicep b/server/main.bicep index 371415e9..8507abe1 100644 --- a/server/main.bicep +++ b/server/main.bicep @@ -71,9 +71,6 @@ param serverImageRepository string = 'tinglesoftware/dependabot-server' @description('Tag of the server docker image.') param serverImageTag string = '#{GITVERSION_NUGETVERSIONV2}#' -@description('Registry and repository of the updater docker image. Ideally, you do not need to edit this value.') -param updaterImageRepository string = 'tinglesoftware/dependabot-updater' - @description('Tag of the updater docker image.') param updaterImageTag string = '#{GITVERSION_NUGETVERSIONV2}#' @@ -360,7 +357,7 @@ resource app 'Microsoft.App/containerApps@2022-10-01' = { } { name: 'Workflow__LogAnalyticsWorkspaceKey', secretRef: 'log-analytics-workspace-key' } { name: 'Workflow__ManagedIdentityId', value: managedIdentityJobs.id } - { name: 'Workflow__UpdaterContainerImage', value: '${'${hasDockerImageRegistry ? '${dockerImageRegistry}/' : ''}'}${updaterImageRepository}:${updaterImageTag}' } + { name: 'Workflow__UpdaterContainerImageTemplate', value: '${'${hasDockerImageRegistry ? '${dockerImageRegistry}/' : ''}'}tinglesoftware/dependabot-updater-{{ecosystem}}:${updaterImageTag}' } { name: 'Workflow__FailOnException', value: failOnException ? 'true' : 'false' } { name: 'Workflow__AutoComplete', value: autoComplete ? 'true' : 'false' } { name: 'Workflow__AutoCompleteIgnoreConfigs', value: join(autoCompleteIgnoreConfigs, ';') } diff --git a/server/main.json b/server/main.json index bbc42cb1..131b259c 100644 --- a/server/main.json +++ b/server/main.json @@ -147,13 +147,6 @@ "description": "Tag of the server docker image." } }, - "updaterImageRepository": { - "type": "string", - "defaultValue": "tinglesoftware/dependabot-updater", - "metadata": { - "description": "Registry and repository of the updater docker image. Ideally, you do not need to edit this value." - } - }, "updaterImageTag": { "type": "string", "defaultValue": "#{GITVERSION_NUGETVERSIONV2}#", @@ -489,8 +482,8 @@ "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', format('{0}-jobs', parameters('name')))]" }, { - "name": "Workflow__UpdaterContainerImage", - "value": "[format('{0}{1}:{2}', format('{0}', if(variables('hasDockerImageRegistry'), format('{0}/', parameters('dockerImageRegistry')), '')), parameters('updaterImageRepository'), parameters('updaterImageTag'))]" + "name": "Workflow__UpdaterContainerImageTemplate", + "value": "[format('{0}tinglesoftware/dependabot-updater-{{{{ecosystem}}}}:{1}', format('{0}', if(variables('hasDockerImageRegistry'), format('{0}/', parameters('dockerImageRegistry')), '')), parameters('updaterImageTag'))]" }, { "name": "Workflow__FailOnException", diff --git a/updater/Dockerfile b/updater/Dockerfile index 98c61c8b..cabbabe1 100644 --- a/updater/Dockerfile +++ b/updater/Dockerfile @@ -1,12 +1,8 @@ -# The tagged versions are currently slow (sometimes it takes months) -# We temporarily switch to getting the gem from git. -# When the changes to this repository are no longer many/major, -# we can switch back to using the tagged versions. +#TODO: find out how to lock the base image version +ARG ECOSYSTEM +FROM ghcr.io/dependabot/dependabot-updater-$ECOSYSTEM -# FROM ghcr.io/dependabot/dependabot-core:0.215.0 -FROM ghcr.io/dependabot/dependabot-core@sha256:3681373aeb07e29fdf30c7a03713195424636fd1cafd569c424a96af27d37735 - -ENV DEPENDABOT_HOME /home/dependabot +# ENV DEPENDABOT_HOME /home/dependabot WORKDIR ${DEPENDABOT_HOME} COPY --chown=dependabot:dependabot updater/Gemfile updater/Gemfile.lock dependabot-updater/ @@ -18,14 +14,6 @@ RUN bundle config set --local path 'vendor' && \ bundle config set --local without 'development' && \ bundle install -# Project files are known to change more frequently than Gemfiles. -# They are copied after installation of dependencies so that the -# image layers that change less frequently are available for caching -# and hence be reused in subsequent builds. -# For more information: -# https://docs.docker.com/develop/develop-images/build_enhancements/ -# https://testdriven.io/blog/faster-ci-builds-with-docker-cache/ - # Add project COPY --chown=dependabot:dependabot LICENSE $DEPENDABOT_HOME COPY --chown=dependabot:dependabot updater $DEPENDABOT_HOME/dependabot-updater