diff --git a/server/Tingle.Dependabot.Tests/Workflow/AzureDevOpsProjectUrlTests.cs b/server/Tingle.Dependabot.Tests/Workflow/AzureDevOpsProjectUrlTests.cs index c93bba37..a1007442 100644 --- a/server/Tingle.Dependabot.Tests/Workflow/AzureDevOpsProjectUrlTests.cs +++ b/server/Tingle.Dependabot.Tests/Workflow/AzureDevOpsProjectUrlTests.cs @@ -15,6 +15,7 @@ public void Creation_WithParsing_Works(string projectUrl, string hostname, strin { var url = (AzureDevOpsProjectUrl)projectUrl; Assert.Equal(hostname, url.Hostname); + Assert.Null(url.Port); Assert.Equal(organizationName, url.OrganizationName); Assert.Equal(organizationUrl, url.OrganizationUrl); Assert.Equal(projectIdOrName, url.ProjectIdOrName); diff --git a/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs b/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs index 899ef754..bd654a54 100644 --- a/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs +++ b/server/Tingle.Dependabot.Tests/Workflow/UpdateRunnerTests.cs @@ -17,7 +17,7 @@ public UpdateRunnerTests(ITestOutputHelper outputHelper) } [Fact] - public void MakeExtraCredentials_Works_1() + public void MakeCredentialsMetadata_Works() { using var stream = TestSamples.GetSampleRegistries(); using var reader = new StreamReader(stream); @@ -28,184 +28,257 @@ public void MakeExtraCredentials_Works_1() var configuration = deserializer.Deserialize(reader); Assert.NotNull(configuration); - var registries = UpdateRunner.MakeExtraCredentials(configuration.Registries.Values, new Dictionary()); - Assert.Equal(11, registries.Count); + var credentials = UpdateRunner.MakeExtraCredentials(configuration.Registries.Values, new Dictionary()); + Assert.Equal(11, credentials.Count); + var metadatas = UpdateRunner.MakeCredentialsMetadata(credentials); + Assert.Equal(11, metadatas.Count); // composer-repository - var registry = registries[0]; - Assert.Equal("composer_repository", Assert.Contains("type", registry)); - Assert.Equal("https://repo.packagist.com/example-company/", Assert.Contains("url", registry)); - Assert.DoesNotContain("registry", registry); - Assert.DoesNotContain("host", registry); - Assert.DoesNotContain("key", registry); - Assert.DoesNotContain("token", registry); - Assert.DoesNotContain("organization", registry); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.Equal("octocat", Assert.Contains("username", registry)); - Assert.Equal("pwd_1234567890", Assert.Contains("password", registry)); - Assert.DoesNotContain("replaces-base", registry); + var metadata = metadatas[0]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "composer_repository", "repo.packagist.com", }, metadata.Values); // docker-registry - registry = registries[1]; - Assert.Equal("docker_registry", Assert.Contains("type", registry)); - Assert.DoesNotContain("url", registry); - Assert.Equal("registry.hub.docker.com", Assert.Contains("registry", registry)); - Assert.DoesNotContain("host", registry); - Assert.DoesNotContain("key", registry); - Assert.DoesNotContain("token", registry); - Assert.DoesNotContain("organization", registry); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.Equal("octocat", Assert.Contains("username", registry)); - Assert.Equal("pwd_1234567890", Assert.Contains("password", registry)); - Assert.Equal("true", Assert.Contains("replaces-base", registry)); + metadata = metadatas[1]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "docker_registry", "registry.hub.docker.com", }, metadata.Values); // git - registry = registries[2]; - Assert.Equal("git", Assert.Contains("type", registry)); - Assert.Equal("https://github.com", Assert.Contains("url", registry)); - Assert.DoesNotContain("registry", registry); - Assert.DoesNotContain("host", registry); - Assert.DoesNotContain("key", registry); - Assert.DoesNotContain("token", registry); - Assert.DoesNotContain("organization", registry); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.Equal("x-access-token", Assert.Contains("username", registry)); - Assert.Equal("pwd_1234567890", Assert.Contains("password", registry)); - Assert.DoesNotContain("replaces-base", registry); + metadata = metadatas[2]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "git", "github.com", }, metadata.Values); // hex-organization - registry = registries[3]; - Assert.Equal("hex_organization", Assert.Contains("type", registry)); - Assert.DoesNotContain("url", registry); - Assert.DoesNotContain("registry", registry); - Assert.DoesNotContain("host", registry); - Assert.Equal("key_1234567890", Assert.Contains("key", registry)); - Assert.DoesNotContain("token", registry); - Assert.Equal("github", Assert.Contains("organization", registry)); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.DoesNotContain("username", registry); - Assert.DoesNotContain("password", registry); - Assert.DoesNotContain("replaces-base", registry); + metadata = metadatas[3]; + Assert.Equal(new[] { "type", }, metadata.Keys); + Assert.Equal(new[] { "hex_organization", }, metadata.Values); // hex-repository - registry = registries[4]; - Assert.Equal("hex_repository", Assert.Contains("type", registry)); - Assert.Equal("https://private-repo.example.com", Assert.Contains("url", registry)); - Assert.DoesNotContain("registry", registry); - Assert.DoesNotContain("host", registry); - Assert.DoesNotContain("key", registry); - Assert.DoesNotContain("token", registry); - Assert.DoesNotContain("organization", registry); - Assert.Equal("private-repo", Assert.Contains("repo", registry)); - Assert.Equal("ak_1234567890", Assert.Contains("auth-key", registry)); - Assert.Equal("pkf_1234567890", Assert.Contains("public-key-fingerprint", registry)); - Assert.DoesNotContain("username", registry); - Assert.DoesNotContain("password", registry); - Assert.DoesNotContain("replaces-base", registry); + metadata = metadatas[4]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "hex_repository", "private-repo.example.com", }, metadata.Values); // maven-repository - registry = registries[5]; - Assert.Equal("maven_repository", Assert.Contains("type", registry)); - Assert.Equal("https://artifactory.example.com", Assert.Contains("url", registry)); - Assert.DoesNotContain("registry", registry); - Assert.DoesNotContain("host", registry); - Assert.DoesNotContain("key", registry); - Assert.DoesNotContain("token", registry); - Assert.DoesNotContain("organization", registry); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.Equal("octocat", Assert.Contains("username", registry)); - Assert.Equal("pwd_1234567890", Assert.Contains("password", registry)); - Assert.Equal("true", Assert.Contains("replaces-base", registry)); + metadata = metadatas[5]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "maven_repository", "artifactory.example.com", }, metadata.Values); // npm-registry - registry = registries[6]; - Assert.Equal("npm_registry", Assert.Contains("type", registry)); - Assert.DoesNotContain("url", registry); - Assert.Equal("npm.pkg.github.com", Assert.Contains("registry", registry)); - Assert.DoesNotContain("host", registry); - Assert.DoesNotContain("key", registry); - Assert.Equal("tkn_1234567890", Assert.Contains("token", registry)); - Assert.DoesNotContain("organization", registry); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.DoesNotContain("username", registry); - Assert.DoesNotContain("password", registry); - Assert.Equal("true", Assert.Contains("replaces-base", registry)); + metadata = metadatas[6]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "npm_registry", "npm.pkg.github.com", }, metadata.Values); // nuget-feed - registry = registries[7]; - Assert.Equal("nuget_feed", Assert.Contains("type", registry)); - Assert.Equal("https://pkgs.dev.azure.com/contoso/_packaging/My_Feed/nuget/v3/index.json", Assert.Contains("url", registry)); - Assert.DoesNotContain("registry", registry); - Assert.DoesNotContain("host", registry); - Assert.DoesNotContain("key", registry); - Assert.DoesNotContain("token", registry); - Assert.DoesNotContain("organization", registry); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.Equal("octocat@example.com", Assert.Contains("username", registry)); - Assert.Equal("pwd_1234567890", Assert.Contains("password", registry)); - Assert.DoesNotContain("replaces-base", registry); + metadata = metadatas[7]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "nuget_feed", "pkgs.dev.azure.com", }, metadata.Values); // python-index - registry = registries[8]; - Assert.Equal("python_index", Assert.Contains("type", registry)); - Assert.Equal("https://pkgs.dev.azure.com/octocat/_packaging/my-feed/pypi/example", Assert.Contains("url", registry)); - Assert.DoesNotContain("registry", registry); - Assert.DoesNotContain("host", registry); - Assert.DoesNotContain("key", registry); - Assert.DoesNotContain("token", registry); - Assert.DoesNotContain("organization", registry); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.Equal("octocat@example.com", Assert.Contains("username", registry)); - Assert.Equal("pwd_1234567890", Assert.Contains("password", registry)); - Assert.Equal("true", Assert.Contains("replaces-base", registry)); + metadata = metadatas[8]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "python_index", "pkgs.dev.azure.com", }, metadata.Values); // rubygems-server - registry = registries[9]; - Assert.Equal("rubygems_server", Assert.Contains("type", registry)); - Assert.Equal("https://rubygems.pkg.github.com/octocat/github_api", Assert.Contains("url", registry)); - Assert.DoesNotContain("registry", registry); - Assert.DoesNotContain("host", registry); - Assert.DoesNotContain("key", registry); - Assert.Equal("tkn_1234567890", Assert.Contains("token", registry)); - Assert.DoesNotContain("organization", registry); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.DoesNotContain("username", registry); - Assert.DoesNotContain("password", registry); - Assert.DoesNotContain("replaces-base", registry); + metadata = metadatas[9]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "rubygems_server", "rubygems.pkg.github.com", }, metadata.Values); // terraform-registry - registry = registries[10]; - Assert.Equal("terraform_registry", Assert.Contains("type", registry)); - Assert.DoesNotContain("url", registry); - Assert.DoesNotContain("registry", registry); - Assert.Equal("terraform.example.com", Assert.Contains("host", registry)); - Assert.DoesNotContain("key", registry); - Assert.Equal("tkn_1234567890", Assert.Contains("token", registry)); - Assert.DoesNotContain("organization", registry); - Assert.DoesNotContain("repo", registry); - Assert.DoesNotContain("auth-key", registry); - Assert.DoesNotContain("public-key-fingerprint", registry); - Assert.DoesNotContain("username", registry); - Assert.DoesNotContain("password", registry); - Assert.DoesNotContain("replaces-base", registry); + metadata = metadatas[10]; + Assert.Equal(new[] { "type", "host", }, metadata.Keys); + Assert.Equal(new[] { "terraform_registry", "terraform.example.com", }, metadata.Values); + } + + [Fact] + public void MakeExtraCredentials_Works() + { + using var stream = TestSamples.GetSampleRegistries(); + using var reader = new StreamReader(stream); + + var deserializer = new DeserializerBuilder().WithNamingConvention(HyphenatedNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + var configuration = deserializer.Deserialize(reader); + Assert.NotNull(configuration); + var credentials = UpdateRunner.MakeExtraCredentials(configuration.Registries.Values, new Dictionary()); + Assert.Equal(11, credentials.Count); + + // composer-repository + var credential = credentials[0]; + Assert.Equal("composer_repository", Assert.Contains("type", credential)); + Assert.Equal("https://repo.packagist.com/example-company/", Assert.Contains("url", credential)); + Assert.DoesNotContain("registry", credential); + Assert.DoesNotContain("host", credential); + Assert.DoesNotContain("key", credential); + Assert.DoesNotContain("token", credential); + Assert.DoesNotContain("organization", credential); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.Equal("octocat", Assert.Contains("username", credential)); + Assert.Equal("pwd_1234567890", Assert.Contains("password", credential)); + Assert.DoesNotContain("replaces-base", credential); + + // docker-registry + credential = credentials[1]; + Assert.Equal("docker_registry", Assert.Contains("type", credential)); + Assert.DoesNotContain("url", credential); + Assert.Equal("registry.hub.docker.com", Assert.Contains("registry", credential)); + Assert.DoesNotContain("host", credential); + Assert.DoesNotContain("key", credential); + Assert.DoesNotContain("token", credential); + Assert.DoesNotContain("organization", credential); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.Equal("octocat", Assert.Contains("username", credential)); + Assert.Equal("pwd_1234567890", Assert.Contains("password", credential)); + Assert.Equal("true", Assert.Contains("replaces-base", credential)); + + // git + credential = credentials[2]; + Assert.Equal("git", Assert.Contains("type", credential)); + Assert.Equal("https://github.com", Assert.Contains("url", credential)); + Assert.DoesNotContain("registry", credential); + Assert.DoesNotContain("host", credential); + Assert.DoesNotContain("key", credential); + Assert.DoesNotContain("token", credential); + Assert.DoesNotContain("organization", credential); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.Equal("x-access-token", Assert.Contains("username", credential)); + Assert.Equal("pwd_1234567890", Assert.Contains("password", credential)); + Assert.DoesNotContain("replaces-base", credential); + + // hex-organization + credential = credentials[3]; + Assert.Equal("hex_organization", Assert.Contains("type", credential)); + Assert.DoesNotContain("url", credential); + Assert.DoesNotContain("registry", credential); + Assert.DoesNotContain("host", credential); + Assert.Equal("key_1234567890", Assert.Contains("key", credential)); + Assert.DoesNotContain("token", credential); + Assert.Equal("github", Assert.Contains("organization", credential)); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.DoesNotContain("username", credential); + Assert.DoesNotContain("password", credential); + Assert.DoesNotContain("replaces-base", credential); + + // hex-repository + credential = credentials[4]; + Assert.Equal("hex_repository", Assert.Contains("type", credential)); + Assert.Equal("https://private-repo.example.com", Assert.Contains("url", credential)); + Assert.DoesNotContain("registry", credential); + Assert.DoesNotContain("host", credential); + Assert.DoesNotContain("key", credential); + Assert.DoesNotContain("token", credential); + Assert.DoesNotContain("organization", credential); + Assert.Equal("private-repo", Assert.Contains("repo", credential)); + Assert.Equal("ak_1234567890", Assert.Contains("auth-key", credential)); + Assert.Equal("pkf_1234567890", Assert.Contains("public-key-fingerprint", credential)); + Assert.DoesNotContain("username", credential); + Assert.DoesNotContain("password", credential); + Assert.DoesNotContain("replaces-base", credential); + + // maven-repository + credential = credentials[5]; + Assert.Equal("maven_repository", Assert.Contains("type", credential)); + Assert.Equal("https://artifactory.example.com", Assert.Contains("url", credential)); + Assert.DoesNotContain("registry", credential); + Assert.DoesNotContain("host", credential); + Assert.DoesNotContain("key", credential); + Assert.DoesNotContain("token", credential); + Assert.DoesNotContain("organization", credential); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.Equal("octocat", Assert.Contains("username", credential)); + Assert.Equal("pwd_1234567890", Assert.Contains("password", credential)); + Assert.Equal("true", Assert.Contains("replaces-base", credential)); + + // npm-registry + credential = credentials[6]; + Assert.Equal("npm_registry", Assert.Contains("type", credential)); + Assert.DoesNotContain("url", credential); + Assert.Equal("npm.pkg.github.com", Assert.Contains("registry", credential)); + Assert.DoesNotContain("host", credential); + Assert.DoesNotContain("key", credential); + Assert.Equal("tkn_1234567890", Assert.Contains("token", credential)); + Assert.DoesNotContain("organization", credential); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.DoesNotContain("username", credential); + Assert.DoesNotContain("password", credential); + Assert.Equal("true", Assert.Contains("replaces-base", credential)); + + // nuget-feed + credential = credentials[7]; + Assert.Equal("nuget_feed", Assert.Contains("type", credential)); + Assert.Equal("https://pkgs.dev.azure.com/contoso/_packaging/My_Feed/nuget/v3/index.json", Assert.Contains("url", credential)); + Assert.DoesNotContain("registry", credential); + Assert.DoesNotContain("host", credential); + Assert.DoesNotContain("key", credential); + Assert.DoesNotContain("token", credential); + Assert.DoesNotContain("organization", credential); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.Equal("octocat@example.com", Assert.Contains("username", credential)); + Assert.Equal("pwd_1234567890", Assert.Contains("password", credential)); + Assert.DoesNotContain("replaces-base", credential); + + // python-index + credential = credentials[8]; + Assert.Equal("python_index", Assert.Contains("type", credential)); + Assert.Equal("https://pkgs.dev.azure.com/octocat/_packaging/my-feed/pypi/example", Assert.Contains("url", credential)); + Assert.DoesNotContain("registry", credential); + Assert.DoesNotContain("host", credential); + Assert.DoesNotContain("key", credential); + Assert.DoesNotContain("token", credential); + Assert.DoesNotContain("organization", credential); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.Equal("octocat@example.com", Assert.Contains("username", credential)); + Assert.Equal("pwd_1234567890", Assert.Contains("password", credential)); + Assert.Equal("true", Assert.Contains("replaces-base", credential)); + + // rubygems-server + credential = credentials[9]; + Assert.Equal("rubygems_server", Assert.Contains("type", credential)); + Assert.Equal("https://rubygems.pkg.github.com/octocat/github_api", Assert.Contains("url", credential)); + Assert.DoesNotContain("registry", credential); + Assert.DoesNotContain("host", credential); + Assert.DoesNotContain("key", credential); + Assert.Equal("tkn_1234567890", Assert.Contains("token", credential)); + Assert.DoesNotContain("organization", credential); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.DoesNotContain("username", credential); + Assert.DoesNotContain("password", credential); + Assert.DoesNotContain("replaces-base", credential); + + // terraform-registry + credential = credentials[10]; + Assert.Equal("terraform_registry", Assert.Contains("type", credential)); + Assert.DoesNotContain("url", credential); + Assert.DoesNotContain("registry", credential); + Assert.Equal("terraform.example.com", Assert.Contains("host", credential)); + Assert.DoesNotContain("key", credential); + Assert.Equal("tkn_1234567890", Assert.Contains("token", credential)); + Assert.DoesNotContain("organization", credential); + Assert.DoesNotContain("repo", credential); + Assert.DoesNotContain("auth-key", credential); + Assert.DoesNotContain("public-key-fingerprint", credential); + Assert.DoesNotContain("username", credential); + Assert.DoesNotContain("password", credential); + Assert.DoesNotContain("replaces-base", credential); } [Fact] @@ -219,4 +292,34 @@ public void ConvertPlaceholder_Works() var result = UpdateRunner.ConvertPlaceholder(input, secrets); Assert.Equal(":cake", result); } + + [Theory] + [MemberData(nameof(ConvertEcosystemToPackageManagerValues))] + public void ConvertEcosystemToPackageManager_Works(string ecosystem, string expected) + { + var actual = UpdateRunner.ConvertEcosystemToPackageManager(ecosystem); + Assert.Equal(expected, actual); + } + + public static TheoryData ConvertEcosystemToPackageManagerValues => new() + { + { "github-actions", "github_actions" }, + { "gitsubmodule", "submodules" }, + { "gomod", "go_modules" }, + { "mix", "hex" }, + { "npm", "npm_and_yarn" }, + { "yarn", "npm_and_yarn" }, + { "pipenv", "pip" }, + { "pip-compile", "pip" }, + { "poetry", "pip" }, + + // retained + { "nuget", "nuget" }, + { "npm", "npm_and_yarn" }, + { "gradle", "gradle" }, + { "maven", "maven" }, + { "swift", "swift" }, + { "terraform", "terraform" }, + { "docker", "docker" }, + }; } diff --git a/server/Tingle.Dependabot/Models/DependabotConfiguration.cs b/server/Tingle.Dependabot/Models/DependabotConfiguration.cs index e3a3690e..70de3e07 100644 --- a/server/Tingle.Dependabot/Models/DependabotConfiguration.cs +++ b/server/Tingle.Dependabot/Models/DependabotConfiguration.cs @@ -53,15 +53,16 @@ public record DependabotUpdate [JsonPropertyName("schedule")] public DependabotUpdateSchedule? Schedule { get; set; } - [Required] [JsonPropertyName("open-pull-requests-limit")] - public int? OpenPullRequestsLimit { get; set; } = 5; + public int OpenPullRequestsLimit { get; set; } = 5; [JsonPropertyName("registries")] public List? Registries { get; set; } [JsonPropertyName("allow")] public List? Allow { get; set; } + [JsonPropertyName("ignore")] + public List? Ignore { get; set; } [JsonPropertyName("labels")] public List? Labels { get; set; } [JsonPropertyName("milestone")] @@ -116,14 +117,40 @@ public string GenerateCron() } } -public class DependabotAllowDependency +public class DependabotAllowDependency : IValidatableObject { [JsonPropertyName("dependency-name")] public string? DependencyName { get; set; } [JsonPropertyName("dependency-type")] public string? DependencyType { get; set; } - public bool IsValid() => DependencyName is not null || DependencyType is not null; + public IEnumerable Validate(ValidationContext validationContext) + { + if (DependencyName is null && DependencyType is null) + { + yield return new ValidationResult("Each entry under 'allow' must have 'dependency-name', 'dependency-type' or both set"); + } + } +} + +public class DependabotIgnoreDependency : IValidatableObject +{ + [JsonPropertyName("dependency-name")] + public string? DependencyName { get; set; } + + [JsonPropertyName("versions")] + public IList? Versions { get; set; } + + [JsonPropertyName("update-types")] + public IList? UpdateTypes { get; set; } + + public IEnumerable Validate(ValidationContext validationContext) + { + if (DependencyName is null && Versions is null && UpdateTypes is null) + { + yield return new ValidationResult("Each entry under 'ignore' must have one of 'dependency-name', 'versions', or 'update-types' set"); + } + } } public class DependabotPullRequestBranchName diff --git a/server/Tingle.Dependabot/Models/UpdateJobResponse.cs b/server/Tingle.Dependabot/Models/DependabotUpdateJobDefinition.cs similarity index 62% rename from server/Tingle.Dependabot/Models/UpdateJobResponse.cs rename to server/Tingle.Dependabot/Models/DependabotUpdateJobDefinition.cs index aee056f3..ddd7ca5b 100644 --- a/server/Tingle.Dependabot/Models/UpdateJobResponse.cs +++ b/server/Tingle.Dependabot/Models/DependabotUpdateJobDefinition.cs @@ -4,82 +4,13 @@ namespace Tingle.Dependabot.Models; -public sealed record UpdateJobResponse(UpdateJobData Data); -public sealed record UpdateJobData(UpdateJobAttributes Attributes); - -public sealed class UpdateJobAttributes +public sealed class DependabotUpdateJobDefinition { - [JsonPropertyName("allowed-updates")] - public required IEnumerable AllowedUpdates { get; set; } - - [JsonPropertyName("credentials-metadata")] - public required IEnumerable CredentialsMetadata { get; set; } - - [JsonPropertyName("dependencies")] - public required IEnumerable Dependencies { get; set; } - - [JsonPropertyName("directory")] - public required string Directory { get; set; } - - [JsonPropertyName("existing-pull-requests")] - public required IEnumerable ExistingPullRequests { get; set; } - - [JsonPropertyName("ignore-conditions")] - public required IEnumerable IgnoreConditions { get; set; } - - [JsonPropertyName("security-advisories")] - public required IEnumerable SecurityAdvisories { get; set; } - - [JsonPropertyName("package_manager")] - public required string PackageManager { get; set; } - - [JsonPropertyName("repo-name")] - public required string RepoName { get; set; } - - [JsonPropertyName("source")] - public required UpdateJobAttributesSource Source { get; set; } - - [JsonPropertyName("lockfile-only")] - public bool? LockfileOnly { get; set; } - - [JsonPropertyName("requirements-update-strategy")] - public string? RequirementsUpdateStrategy { get; set; } - - [JsonPropertyName("update-subdependencies")] - public bool? UpdateSubdependencies { get; set; } - - [JsonPropertyName("updating-a-pull-request")] - public bool? UpdatingAPullRequest { get; set; } - - [JsonPropertyName("vendor-dependencies")] - public bool? VendorDependencies { get; set; } - - [JsonPropertyName("security-updates-only")] - public bool? SecurityUpdatesOnly { get; set; } - - [JsonPropertyName("debug")] - public bool? Debug { get; set; } -} - -public sealed class UpdateJobAttributesSource -{ - [JsonPropertyName("provider")] - public required string Provider { get; set; } - - [JsonPropertyName("repo")] - public required string Repo { get; set; } - - [JsonPropertyName("directory")] - public required string Directory { get; set; } - - [JsonPropertyName("branch")] - public string? Branch { get; set; } - - [JsonPropertyName("hostname")] - public string? Hostname { get; set; } + [JsonPropertyName("job")] + public required JsonObject? Job { get; set; } - [JsonPropertyName("api-endpoint")] - public string? ApiEndpoint { get; set; } + [JsonPropertyName("credentials")] + public required JsonArray Credentials { get; set; } } public sealed class CreatePullRequestModel diff --git a/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs b/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs index 9f555150..73a7a69c 100644 --- a/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs +++ b/server/Tingle.Dependabot/Workflow/AzureDevOpsProjectUrl.cs @@ -15,6 +15,12 @@ public AzureDevOpsProjectUrl(Uri uri) { this.uri = uri ?? throw new ArgumentNullException(nameof(uri)); var host = Hostname = uri.Host; + Port = uri switch + { + { Scheme: "http", Port: 80 } => null, + { Scheme: "https", Port: 443 } => null, + _ => uri.Port, + }; var builder = new UriBuilder(uri) { UserName = null, Password = null }; if (string.Equals(host, "dev.azure.com", StringComparison.OrdinalIgnoreCase)) @@ -29,6 +35,7 @@ public AzureDevOpsProjectUrl(Uri uri) builder.Path = string.Empty; ProjectIdOrName = uri.AbsolutePath.Replace("_apis/projects/", "").Split("/")[1]; } + // TODO: add support for Azure DevOps Server here else throw new ArgumentException($"Error parsing: '{uri}' into components"); OrganizationUrl = builder.Uri.ToString(); @@ -52,6 +59,7 @@ public static AzureDevOpsProjectUrl Create(string hostname, string organizationN } public string Hostname { get; } + public int? Port { get; } public string OrganizationName { get; } public string OrganizationUrl { get; } public string ProjectIdOrName { get; } diff --git a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs index 14db5727..d39f49b7 100644 --- a/server/Tingle.Dependabot/Workflow/UpdateRunner.cs +++ b/server/Tingle.Dependabot/Workflow/UpdateRunner.cs @@ -5,6 +5,7 @@ using Azure.ResourceManager.AppContainers.Models; using Azure.ResourceManager.Resources; using Microsoft.Extensions.Options; +using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; @@ -18,6 +19,7 @@ internal partial class UpdateRunner private static partial Regex PlaceholderPattern(); private const string UpdaterContainerName = "updater"; + private const string JobDefinitionFileName = "job.json"; private static readonly JsonSerializerOptions serializerOptions = new(JsonSerializerDefaults.Web); @@ -50,6 +52,12 @@ public async Task CreateAsync(Repository repository, RepositoryUpdate update, Up } catch (Azure.RequestFailedException rfe) when (rfe.Status is 404) { } + // prepare credentials with replaced secrets + var secrets = new Dictionary(options.Secrets) { ["DEFAULT_TOKEN"] = options.ProjectToken!, }; + var registries = update.Registries?.Select(r => repository.Registries[r]).ToList(); + var credentials = MakeExtraCredentials(registries, secrets); // add source credentials when running the in v2 + var directory = Path.Join(options.WorkingDirectory, job.Id); + // prepare the container var volumeName = "working-dir"; var container = new ContainerAppContainer @@ -58,9 +66,9 @@ public async Task CreateAsync(Repository repository, RepositoryUpdate update, Up Image = options.UpdaterContainerImageTemplate!.Replace("{{ecosystem}}", job.PackageEcosystem), Resources = job.Resources!, Args = { "update_script", }, - VolumeMounts = { new ContainerAppVolumeMount { VolumeName = volumeName, MountPath = "/mnt/dependabot", }, }, + VolumeMounts = { new ContainerAppVolumeMount { VolumeName = volumeName, MountPath = options.WorkingDirectory, }, }, }; - var env = CreateVariables(repository, update, job); + var env = CreateEnvironmentVariables(repository, update, job, directory, credentials); foreach (var (key, value) in env) container.Env.Add(new ContainerAppEnvironmentVariable { Name = key, Value = value, }); // prepare the ContainerApp job @@ -96,12 +104,16 @@ public async Task CreateAsync(Repository repository, RepositoryUpdate update, Up { ["purpose"] = "dependabot", ["ecosystem"] = job.PackageEcosystem, - ["repository"] = repository.Slug, - ["directory"] = update.Directory, + ["repository"] = job.RepositorySlug, + ["directory"] = job.Directory, ["machine-name"] = Environment.MachineName, }, }; + // write job definition file + var jobDefinitionPath = await WriteJobDefinitionAsync(update, job, directory, credentials, cancellationToken); + logger.LogInformation("Written job definition file at {JobDefinitionPath}", jobDefinitionPath); + // create the ContainerApp Job var operation = await containerAppJobs.CreateOrUpdateAsync(Azure.WaitUntil.Completed, resourceName, data, cancellationToken); logger.LogInformation("Created ContainerApp Job for {UpdateJobId}", job.Id); @@ -202,71 +214,46 @@ public async Task DeleteAsync(UpdateJob job, CancellationToken cancellationToken internal static string MakeResourceName(UpdateJob job) => $"dependabot-{job.Id}"; - internal IDictionary CreateVariables(Repository repository, RepositoryUpdate update, UpdateJob job) + internal IDictionary CreateEnvironmentVariables(Repository repository, + RepositoryUpdate update, + UpdateJob job, + string directory, + IList> credentials) // TODO: unit test this { - static string? ToJson(T? entries) => entries is null ? null : JsonSerializer.Serialize(entries, serializerOptions); // null ensures we do not add to the values - - // Prepare extra credentials with replaced secrets - var secrets = new Dictionary(options.Secrets) { ["DEFAULT_TOKEN"] = options.ProjectToken!, }; - var registries = update.Registries?.Select(r => repository.Registries[r]).ToList(); - var credentials = ToJson(MakeExtraCredentials(registries, secrets)); // add source credentials when running the in v2 - - var jobDirectory = Path.Join(options.WorkingDirectory, job.Id); - - //var attr = new UpdateJobAttributes - //{ - // AllowedUpdates = Array.Empty(), - // CredentialsMetadata = Array.Empty(), - // Dependencies = Array.Empty(), - // Directory = job.Directory!, - // ExistingPullRequests = Array.Empty(), - // IgnoreConditions = Array.Empty(), - // PackageManager = job.PackageEcosystem!, - // RepoName = job.RepositorySlug!, - // SecurityAdvisories = Array.Empty(), - // Source = new UpdateJobAttributesSource - // { - // Directory = job.Directory!, - // Provider = "azure", - // Repo = job.RepositorySlug!, - // Branch = update.TargetBranch, - // Hostname = , - // ApiEndpoint =, - // }, - //}; - - // TODO: write the job definition file (find out if it is YAML/JSON) + [return: NotNullIfNotNull(nameof(value))] + static string? ToJson(T? value) => value is null ? null : JsonSerializer.Serialize(value, serializerOptions); // null ensures we do not add to the values // Add compulsory values var values = new Dictionary { + // env for v2 ["DEPENDABOT_JOB_ID"] = job.Id!, ["DEPENDABOT_JOB_TOKEN"] = job.AuthKey!, - ["DEPENDABOT_JOB_PATH"] = Path.Join(jobDirectory, "job.json"), - ["DEPENDABOT_OUTPUT_PATH"] = Path.Join(jobDirectory, "output"), - + ["DEPENDABOT_DEBUG"] = (options.DebugJobs ?? false).ToString().ToLower(), + ["DEPENDABOT_API_URL"] = options.JobsApiUrl!, + ["DEPENDABOT_JOB_PATH"] = Path.Join(directory, JobDefinitionFileName), + ["DEPENDABOT_OUTPUT_PATH"] = Path.Join(directory, "output"), + // Setting DEPENDABOT_REPO_CONTENTS_PATH causes some issues, ignore till we can resolve + //["DEPENDABOT_REPO_CONTENTS_PATH"] = Path.Join(jobDirectory, "repo"), + ["GITHUB_ACTIONS"] = "false", + ["UPDATER_DETERMINISTIC"] = (options.DeterministicUpdates ?? false).ToString().ToLower(), + + // env for v1 ["DEPENDABOT_PACKAGE_MANAGER"] = job.PackageEcosystem!, - ["DEPENDABOT_DIRECTORY"] = update.Directory!, - ["DEPENDABOT_OPEN_PULL_REQUESTS_LIMIT"] = update.OpenPullRequestsLimit!.Value.ToString(), - - ["DEPENDABOT_EXTRA_CREDENTIALS"] = credentials!, - ["DEPENDABOT_FAIL_ON_EXCEPTION"] = "false", + ["DEPENDABOT_DIRECTORY"] = job.Directory!, + ["DEPENDABOT_OPEN_PULL_REQUESTS_LIMIT"] = update.OpenPullRequestsLimit.ToString(), + ["DEPENDABOT_EXTRA_CREDENTIALS"] = ToJson(credentials), + ["DEPENDABOT_FAIL_ON_EXCEPTION"] = "false", // we the script to run to completion so that we get notified of job completion }; // Add optional values - values.AddIfNotDefault("DEPENDABOT_DEBUG", options.DebugJobs?.ToString().ToLower()) - .AddIfNotDefault("DEPENDABOT_API_URL", options.JobsApiUrl) - // Setting DEPENDABOT_REPO_CONTENTS_PATH causes some issues, ignore till we can resolve - //.AddIfNotDefault("DEPENDABOT_REPO_CONTENTS_PATH", Path.Join(jobDirectory, "repo")) - .AddIfNotDefault("UPDATER_DETERMINISTIC", options.DeterministicUpdates?.ToString().ToLower()); - values.AddIfNotDefault("GITHUB_ACCESS_TOKEN", options.GithubToken) .AddIfNotDefault("DEPENDABOT_REBASE_STRATEGY", update.RebaseStrategy) .AddIfNotDefault("DEPENDABOT_TARGET_BRANCH", update.TargetBranch) .AddIfNotDefault("DEPENDABOT_VENDOR", update.Vendor ? "true" : null) .AddIfNotDefault("DEPENDABOT_REJECT_EXTERNAL_CODE", string.Equals(update.InsecureExternalCodeExecution, "deny").ToString().ToLowerInvariant()) .AddIfNotDefault("DEPENDABOT_VERSIONING_STRATEGY", update.VersioningStrategy) - .AddIfNotDefault("DEPENDABOT_ALLOW_CONDITIONS", ToJson(MakeAllowEntries(update.Allow))) + .AddIfNotDefault("DEPENDABOT_ALLOW_CONDITIONS", ToJson(update.Allow)) .AddIfNotDefault("DEPENDABOT_LABELS", ToJson(update.Labels)) .AddIfNotDefault("DEPENDABOT_BRANCH_NAME_SEPARATOR", update.PullRequestBranchName?.Separator) .AddIfNotDefault("DEPENDABOT_MILESTONE", update.Milestone?.ToString()); @@ -285,15 +272,99 @@ internal IDictionary CreateVariables(Repository repository, Repo return values; } - internal static IList> MakeExtraCredentials(ICollection? registries, IDictionary secrets) + + internal async Task WriteJobDefinitionAsync(RepositoryUpdate update, + UpdateJob job, + string directory, + IList> credentials, + CancellationToken cancellationToken = default) // TODO: unit test this + { + [return: NotNullIfNotNull(nameof(value))] + static JsonNode? ToJsonNode(T? value) => value is null ? null : JsonSerializer.SerializeToNode(value, serializerOptions); // null ensures we do not add to the values + + var url = options.ProjectUrl!.Value; + var credentialsMetadata = MakeCredentialsMetadata(credentials); + + var definition = new DependabotUpdateJobDefinition + { + Job = new JsonObject + { + ["allowed-updates"] = ToJsonNode(update.Allow ?? new()), + ["credentials-metadata"] = ToJsonNode(credentialsMetadata).AsArray(), + // ["dependencies"] = null, // object array + ["directory"] = job.Directory, + // ["existing-pull-requests"] = null, // object array + ["ignore-conditions"] = ToJsonNode(update.Ignore ?? new()), + // ["security-advisories"] = null, // object array + ["package_manager"] = ConvertEcosystemToPackageManager(job.PackageEcosystem!), + ["repo-name"] = job.RepositorySlug, + ["source"] = new JsonObject + { + ["provider"] = "azure", + ["repo"] = job.RepositorySlug, + ["directory"] = job.Directory, + ["branch"] = update.TargetBranch, + ["hostname"] = url.Hostname, + ["api-endpoint"] = new UriBuilder + { + Scheme = Uri.UriSchemeHttps, + Host = url.Hostname, + Port = url.Port ?? -1, + }.ToString(), + }, + ["lockfile-only"] = update.VersioningStrategy == "lockfile-only", + ["requirements-update-strategy"] = update.VersioningStrategy?.Replace("-", "_"), + // ["update-subdependencies"] = false, + // ["updating-a-pull-request"] = false, + ["vendor-dependencies"] = update.Vendor, + ["security-updates-only"] = update.OpenPullRequestsLimit == 0, + ["debug"] = false, + }, + Credentials = ToJsonNode(credentials).AsArray(), + }; + + // write the job definition file + var path = Path.Join(directory, JobDefinitionFileName); + if (File.Exists(path)) File.Delete(path); + using var stream = File.OpenWrite(path); + await JsonSerializer.SerializeAsync(stream, definition, serializerOptions, cancellationToken); + + return path; + } + + internal static IList> MakeCredentialsMetadata(IList> credentials) { - if (registries is null) return Array.Empty>(); + return credentials.Select(cred => + { + var values = new Dictionary { ["type"] = cred["type"], }; + cred.TryGetValue("host", out var host); + + // pull host from registry if available + if (string.IsNullOrWhiteSpace(host)) + { + host = cred.TryGetValue("registry", out var registry) && Uri.TryCreate($"https://{registry}", UriKind.Absolute, out var u) ? u.Host : host; + } + + // pull host from registry if url + if (string.IsNullOrWhiteSpace(host)) + { + host = cred.TryGetValue("url", out var url) && Uri.TryCreate(url, UriKind.Absolute, out var u) ? u.Host : host; + } + + values.AddIfNotDefault("host", host); + + return values; + }).ToList(); + } + internal static IList> MakeExtraCredentials(ICollection? registries, IDictionary secrets) + { + if (registries is null) return Array.Empty>(); return registries.Select(v => { var type = v.Type?.Replace("-", "_") ?? throw new InvalidOperationException("Type should not be null"); - var values = new Dictionary().AddIfNotDefault("type", type); + var values = new Dictionary { ["type"] = type, }; // values for hex-organization values.AddIfNotDefault("organization", v.Organization); @@ -350,13 +421,24 @@ internal static IList> MakeExtraCredentials(ICollect return result; } - internal static IList>? MakeAllowEntries(List? entries) + internal static string? ConvertEcosystemToPackageManager(string ecosystem) { - return entries?.Where(e => e.IsValid()) - .Select(e => new Dictionary() - .AddIfNotDefault("dependency-name", e.DependencyName) - .AddIfNotDefault("dependency-type", e.DependencyType)) - .ToList(); + ArgumentException.ThrowIfNullOrEmpty(ecosystem); + + return ecosystem switch + { + "github-actions" => "github_actions", + "gitsubmodule" => "submodules", + "gomod" => "go_modules", + "mix" => "hex", + "npm" => "npm_and_yarn", + // Additional ones + "yarn" => "npm_and_yarn", + "pipenv" => "pip", + "pip-compile" => "pip", + "poetry" => "pip", + _ => ecosystem, + }; } }