Add more support for Azure DevOps

.env.example

@@ -0,0 +1,5 @@
+LOCAL_AZURE_ACCESS_TOKEN=
+LOCAL_GITHUB_ACCESS_TOKEN=
+DEPENDABOT_PROVIDER=
+DEPENDABOT_PACKAGE_MANAGER=
+DEPENDABOT_REPO=

.gitignore

@@ -3,3 +3,4 @@ testdata/caches
 cache
 out.yaml
 ./dependabot
+.env

cmd/dependabot/internal/cmd/test.go

@@ -32,7 +32,7 @@ func NewTestCommand() *cobra.Command {
 				return err
 			}
 
-			processInput(&scenario.Input)
+			processInput(&scenario.Input, nil)
 
 			if err := executeTestJob(infra.RunParams{
 				CacheDir:            flags.cache,

cmd/dependabot/internal/cmd/update.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"log"
 	"net"
+	"net/url"
 	"os"
 
 	"github.com/MakeNowJust/heredoc"
@@ -32,6 +33,7 @@ type UpdateFlags struct {
 	commit          string
 	dependencies    []string
 	inputServerPort int
+	apiUrl          string
 }
 
 func NewUpdateCommand() *cobra.Command {
@@ -60,7 +62,10 @@ func NewUpdateCommand() *cobra.Command {
 				return err
 			}
 
-			processInput(input)
+			err = processInput(input, &flags)
+			if err != nil {
+				return err
+			}
 
 			var writer io.Writer
 			if !flags.debugging {
@@ -86,6 +91,7 @@ func NewUpdateCommand() *cobra.Command {
 				UpdaterImage:        updaterImage,
 				Volumes:             flags.volumes,
 				Writer:              writer,
+				ApiUrl:              flags.apiUrl,
 			}); err != nil {
 				log.Fatalf("failed to run updater: %v", err)
 			}
@@ -112,6 +118,7 @@ func NewUpdateCommand() *cobra.Command {
 	cmd.Flags().StringArrayVar(&flags.extraHosts, "extra-hosts", nil, "Docker extra hosts setting on the proxy")
 	cmd.Flags().DurationVarP(&flags.timeout, "timeout", "t", 0, "max time to run an update")
 	cmd.Flags().IntVar(&flags.inputServerPort, "input-port", 0, "port to use for securely passing input to the updater")
+	cmd.Flags().StringVarP(&flags.apiUrl, "api-url", "a", "", "the api dependabot should connect to.")
 
 	return cmd
 }
@@ -238,7 +245,7 @@ func readInputFile(file string) (*model.Input, error) {
 	return &input, nil
 }
 
-func processInput(input *model.Input) {
+func processInput(input *model.Input, flags *UpdateFlags) error {
 	job := &input.Job
 	// a few of the fields need to be initialized instead of null,
 	// it would be nice if the updater didn't care
@@ -258,16 +265,30 @@ func processInput(input *model.Input) {
 		job.DependencyGroups = []model.Group{}
 	}
 
+	azureRepo := model.NewAzureRepo(input.Job.PackageManager, input.Job.Source.Repo, input.Job.Source.Directory)
+
 	// As a convenience, fill in a git_source if credentials are in the environment and a git_source
 	// doesn't already exist. This way the user doesn't run out of calls from being anonymous.
 	hasLocalToken := os.Getenv("LOCAL_GITHUB_ACCESS_TOKEN") != ""
+	hasLocalAzureToken := os.Getenv("LOCAL_AZURE_ACCESS_TOKEN") != ""
+
 	var isGitSourceInCreds bool
 	for _, cred := range input.Credentials {
 		if cred["type"] == "git_source" {
 			isGitSourceInCreds = true
 			break
 		}
 	}
+	if hasLocalAzureToken && flags.apiUrl != "" && azureRepo != nil {
+		u, _ := url.Parse(flags.apiUrl)
+		input.Credentials = append(input.Credentials, model.Credential{
+			"type":     "git_source",
+			"host":     u.Hostname(),
+			"username": azureRepo.Org,
+			"password": "$LOCAL_AZURE_ACCESS_TOKEN",
+		})
+	}
+
 	if hasLocalToken && !isGitSourceInCreds {
 		log.Println("Inserting $LOCAL_GITHUB_ACCESS_TOKEN into credentials")
 		input.Credentials = append(input.Credentials, model.Credential{
@@ -285,6 +306,50 @@ func processInput(input *model.Input) {
 		}
 	}
 
+	if hasLocalAzureToken && !isGitSourceInCreds && azureRepo != nil {
+		log.Println("Inserting $LOCAL_AZURE_ACCESS_TOKEN into credentials")
+		log.Printf("Inserting artifacts credentials for %s organization.", azureRepo.Org)
+		input.Credentials = append(input.Credentials, model.Credential{
+			"type":     "git_source",
+			"host":     "dev.azure.com",
+			"username": "x-access-token",
+			"password": "$LOCAL_AZURE_ACCESS_TOKEN",
+		})
+		if len(input.Job.CredentialsMetadata) > 0 {
+			// Add the metadata since the next section will be skipped.
+			input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, map[string]any{
+				"type": "git_source",
+				"host": "dev.azure.com",
+			})
+		}
+		input.Credentials = append(input.Credentials, model.Credential{
+			"type":     "git_source",
+			"host":     fmt.Sprintf("%s.pkgs.visualstudio.com", azureRepo.Org),
+			"username": "x-access-token",
+			"password": "$LOCAL_AZURE_ACCESS_TOKEN",
+		})
+		if len(input.Job.CredentialsMetadata) > 0 {
+			// Add the metadata since the next section will be skipped.
+			input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, map[string]any{
+				"type": "git_source",
+				"host": fmt.Sprintf("%s.pkgs.visualstudio.com", azureRepo.Org),
+			})
+		}
+		input.Credentials = append(input.Credentials, model.Credential{
+			"type":     "git_source",
+			"host":     "pkgs.dev.azure.com",
+			"username": "x-access-token",
+			"password": "$LOCAL_AZURE_ACCESS_TOKEN",
+		})
+		if len(input.Job.CredentialsMetadata) > 0 {
+			// Add the metadata since the next section will be skipped.
+			input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, map[string]any{
+				"type": "git_source",
+				"host": "pkgs.dev.azure.com",
+			})
+		}
+	}
+
 	// As a convenience, fill credentials-metadata if credentials are provided
 	// which is what happens in production. This way the user doesn't have to
 	// specify credentials-metadata in the scenario file unless they want to.
@@ -309,6 +374,8 @@ func processInput(input *model.Input) {
 			input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, entry)
 		}
 	}
+
+	return nil
 }
 
 func doesStdinHaveData() bool {

go.mod

@@ -14,6 +14,11 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+)
+
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.6.0 // indirect
@@ -35,6 +40,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.4
 	golang.org/x/mod v0.7.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect

go.sum

@@ -82,9 +82,12 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

internal/infra/run.go

@@ -68,6 +68,7 @@ type RunParams struct {
 	Writer    io.Writer
 	InputName string
 	InputRaw  []byte
+	ApiUrl    string
 }
 
 var gitShaRegex = regexp.MustCompile(`^[0-9a-f]{40}$`)
@@ -79,6 +80,9 @@ func (p *RunParams) Validate() error {
 	if p.Job.Source.Commit != "" && !gitShaRegex.MatchString(p.Job.Source.Commit) {
 		return fmt.Errorf("commit must be a SHA, or not provided")
 	}
+	if p.ApiUrl == "" {
+		p.ApiUrl = fmt.Sprintf("http://host.docker.internal:%v", api.Port())
+	}
 	return nil
 }
 
@@ -125,7 +129,7 @@ func Run(params RunParams) error {
 		return err
 	}
 
-	if err := runContainers(ctx, params, api); err != nil {
+	if err := runContainers(ctx, params); err != nil {
 		return err
 	}
 
@@ -277,16 +281,18 @@ func setImageNames(params *RunParams) error {
 }
 
 func expandEnvironmentVariables(api *server.API, params *RunParams) {
-	api.Actual.Input.Credentials = params.Creds
-
-	// Make a copy of the credentials, so we don't inject them into the output file.
-	params.Creds = []model.Credential{}
-	for _, cred := range api.Actual.Input.Credentials {
-		newCred := model.Credential{}
-		for k, v := range cred {
-			newCred[k] = v
+	if api != nil {
+		api.Actual.Input.Credentials = params.Creds
+
+		// Make a copy of the credentials, so we don't inject them into the output file.
+		params.Creds = []model.Credential{}
+		for _, cred := range api.Actual.Input.Credentials {
+			newCred := model.Credential{}
+			for k, v := range cred {
+				newCred[k] = v
+			}
+			params.Creds = append(params.Creds, newCred)
 		}
-		params.Creds = append(params.Creds, newCred)
 	}
 
 	// Add the actual credentials from the environment.
@@ -324,7 +330,7 @@ func generateIgnoreConditions(params *RunParams, actual *model.Scenario) error {
 	return nil
 }
 
-func runContainers(ctx context.Context, params RunParams, api *server.API) error {
+func runContainers(ctx context.Context, params RunParams) error {
 	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
 	if err != nil {
 		return fmt.Errorf("failed to create Docker client: %w", err)
@@ -389,12 +395,12 @@ func runContainers(ctx context.Context, params RunParams, api *server.API) error
 	}
 
 	if params.Debug {
-		if err := updater.RunShell(ctx, prox.url, api.Port()); err != nil {
+		if err := updater.RunShell(ctx, prox.url, params.ApiUrl); err != nil {
 			return err
 		}
 	} else {
 		const cmd = "update-ca-certificates && bin/run fetch_files && bin/run update_files"
-		if err := updater.RunCmd(ctx, cmd, dependabot, userEnv(prox.url, api.Port())...); err != nil {
+		if err := updater.RunCmd(ctx, cmd, dependabot, userEnv(prox.url, params.ApiUrl)...); err != nil {
 			return err
 		}
 	}

internal/infra/updater.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/goware/prefixer"
 	"io"
 	"os"
 	"path"
@@ -19,6 +18,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/network"
+	"github.com/goware/prefixer"
 	"github.com/moby/moby/client"
 	"github.com/moby/moby/pkg/stdcopy"
 )
@@ -153,34 +153,34 @@ func mountOptions(v string) (local, remote string, readOnly bool, err error) {
 	return local, remote, readOnly, nil
 }
 
-func userEnv(proxyURL string, apiPort int) []string {
+func userEnv(proxyURL string, apiUrl string) []string {
 	return []string{
 		"GITHUB_ACTIONS=true", // sets exit code when fetch fails
 		fmt.Sprintf("http_proxy=%s", proxyURL),
 		fmt.Sprintf("HTTP_PROXY=%s", proxyURL),
 		fmt.Sprintf("https_proxy=%s", proxyURL),
 		fmt.Sprintf("HTTPS_PROXY=%s", proxyURL),
-		fmt.Sprintf("DEPENDABOT_JOB_ID=%v", jobID),
+		fmt.Sprintf("DEPENDABOT_JOB_ID=%v", firstNonEmpty(os.Getenv("DEPENDABOT_JOB_ID"), jobID)),
 		fmt.Sprintf("DEPENDABOT_JOB_TOKEN=%v", ""),
 		fmt.Sprintf("DEPENDABOT_JOB_PATH=%v", guestInputDir),
 		fmt.Sprintf("DEPENDABOT_OUTPUT_PATH=%v", guestOutput),
 		fmt.Sprintf("DEPENDABOT_REPO_CONTENTS_PATH=%v", guestRepoDir),
-		fmt.Sprintf("DEPENDABOT_API_URL=http://host.docker.internal:%v", apiPort),
+		fmt.Sprintf("DEPENDABOT_API_URL=%", apiUrl),
 		fmt.Sprintf("SSL_CERT_FILE=%v/ca-certificates.crt", certsPath),
 		"UPDATER_ONE_CONTAINER=true",
 		"UPDATER_DETERMINISTIC=true",
 	}
 }
 
 // RunShell executes an interactive shell, blocks until complete.
-func (u *Updater) RunShell(ctx context.Context, proxyURL string, apiPort int) error {
+func (u *Updater) RunShell(ctx context.Context, proxyURL string, apiUrl string) error {
 	execCreate, err := u.cli.ContainerExecCreate(ctx, u.containerID, types.ExecConfig{
 		AttachStdin:  true,
 		AttachStdout: true,
 		AttachStderr: true,
 		Tty:          true,
 		User:         dependabot,
-		Env:          append(userEnv(proxyURL, apiPort), "DEBUG=1"),
+		Env:          append(userEnv(proxyURL, apiUrl), "DEBUG=1"),
 		Cmd:          []string{"/bin/bash", "-c", "update-ca-certificates && /bin/bash"},
 	})
 	if err != nil {
@@ -326,3 +326,13 @@ func addFileToArchive(tw *tar.Writer, name string, mode int64, content string) e
 
 	return nil
 }
+
+func firstNonEmpty(values ...string) string {
+	for _, v := range values {
+		if v != "" {
+			return v
+		}
+	}
+
+	return ""
+}

internal/model/azure.go

@@ -0,0 +1,31 @@
+package model
+
+import "strings"
+
+type AzureRepo struct {
+	PackageManger string
+	Org           string
+	Project       string
+	Repo          string
+	Directory     string
+}
+
+// NewAzureRepo parses a repo string and returns an AzureRepo struct
+// Expects a repo string in the format org/project/repo
+func NewAzureRepo(packageManager string, repo string, directory string) *AzureRepo {
+	repoParts := strings.Split(repo, "/")
+	for i, part := range repoParts {
+		println(i, part)
+	}
+	if len(repoParts) != 3 {
+		return nil
+	}
+
+	return &AzureRepo{
+		PackageManger: packageManager,
+		Org:           repoParts[0],
+		Project:       repoParts[1],
+		Repo:          repoParts[2],
+		Directory:     directory,
+	}
+}