Skip to content

Commit

Permalink
Support project-level Terraform distribution selection
Browse files Browse the repository at this point in the history 
Add support for terraform_distribution config value in project config.
This config value behaves similarly to terraform_version whereby defaults
taken from the server config will be overridden by project-level values.

Also refactor to prevent terraform client slightly to prevent
cyclical dependencies.

Signed-off-by: Andrew Borg <[email protected]>
  • Loading branch information
@abborg
abborg committed Dec 15, 2024
1 parent af0e9dd commit 20e0cc3
Show file tree
Hide file tree
Showing 51 changed files with 728 additions and 430 deletions.
66 changes: 40 additions & 26 deletions cmd/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ const (
CheckoutStrategyFlag = "checkout-strategy"
ConfigFlag = "config"
DataDirFlag = "data-dir"
DefaultTFDistributionFlag = "default-tf-distribution"
DefaultTFVersionFlag = "default-tf-version"
DisableApplyAllFlag = "disable-apply-all"
DisableAutoplanFlag = "disable-autoplan"
Expand Down Expand Up @@ -141,21 +142,22 @@ const (
SSLCertFileFlag = "ssl-cert-file"
SSLKeyFileFlag = "ssl-key-file"
RestrictFileList = "restrict-file-list"
TFDistributionFlag = "tf-distribution"
TFDownloadFlag = "tf-download"
TFDownloadURLFlag = "tf-download-url"
UseTFPluginCache = "use-tf-plugin-cache"
VarFileAllowlistFlag = "var-file-allowlist"
VCSStatusName = "vcs-status-name"
IgnoreVCSStatusNames = "ignore-vcs-status-names"
TFEHostnameFlag = "tfe-hostname"
TFELocalExecutionModeFlag = "tfe-local-execution-mode"
TFETokenFlag = "tfe-token"
WriteGitCredsFlag = "write-git-creds" // nolint: gosec
WebBasicAuthFlag = "web-basic-auth"
WebUsernameFlag = "web-username"
WebPasswordFlag = "web-password"
WebsocketCheckOrigin = "websocket-check-origin"
// TFDistributionFlag is deprecated for DefaultTFDistributionFlag
TFDistributionFlag = "tf-distribution"
TFDownloadFlag = "tf-download"
TFDownloadURLFlag = "tf-download-url"
UseTFPluginCache = "use-tf-plugin-cache"
VarFileAllowlistFlag = "var-file-allowlist"
VCSStatusName = "vcs-status-name"
IgnoreVCSStatusNames = "ignore-vcs-status-names"
TFEHostnameFlag = "tfe-hostname"
TFELocalExecutionModeFlag = "tfe-local-execution-mode"
TFETokenFlag = "tfe-token"
WriteGitCredsFlag = "write-git-creds" // nolint: gosec
WebBasicAuthFlag = "web-basic-auth"
WebUsernameFlag = "web-username"
WebPasswordFlag = "web-password"
WebsocketCheckOrigin = "websocket-check-origin"

// NOTE: Must manually set these as defaults in the setDefaults function.
DefaultADBasicUser = ""
Expand Down Expand Up @@ -421,8 +423,8 @@ var stringFlags = map[string]stringFlag{
description: fmt.Sprintf("File containing x509 private key matching --%s.", SSLCertFileFlag),
},
TFDistributionFlag: {
description: fmt.Sprintf("Which TF distribution to use. Can be set to %s or %s.", TFDistributionTerraform, TFDistributionOpenTofu),
defaultValue: DefaultTFDistribution,
description: "[Deprecated for --default-tf-distribution].",
hidden: true,
},
TFDownloadURLFlag: {
description: "Base URL to download Terraform versions from.",
Expand All @@ -437,6 +439,10 @@ var stringFlags = map[string]stringFlag{
" Only set if using TFC/E as a remote backend." +
" Should be specified via the ATLANTIS_TFE_TOKEN environment variable for security.",
},
DefaultTFDistributionFlag: {
description: fmt.Sprintf("Which TF distribution to use. Can be set to %s or %s.", TFDistributionTerraform, TFDistributionOpenTofu),
defaultValue: DefaultTFDistribution,
},
DefaultTFVersionFlag: {
description: "Terraform version to default to (ex. v0.12.0). Will download if not yet on disk." +
" If not set, Atlantis uses the terraform binary in its PATH.",
Expand Down Expand Up @@ -840,12 +846,13 @@ func (s *ServerCmd) run() error {

// Config looks good. Start the server.
server, err := s.ServerCreator.NewServer(userConfig, server.Config{
AllowForkPRsFlag: AllowForkPRsFlag,
AtlantisURLFlag: AtlantisURLFlag,
AtlantisVersion: s.AtlantisVersion,
DefaultTFVersionFlag: DefaultTFVersionFlag,
RepoConfigJSONFlag: RepoConfigJSONFlag,
SilenceForkPRErrorsFlag: SilenceForkPRErrorsFlag,
AllowForkPRsFlag: AllowForkPRsFlag,
AtlantisURLFlag: AtlantisURLFlag,
AtlantisVersion: s.AtlantisVersion,
DefaultTFDistributionFlag: DefaultTFDistributionFlag,
DefaultTFVersionFlag: DefaultTFVersionFlag,
RepoConfigJSONFlag: RepoConfigJSONFlag,
SilenceForkPRErrorsFlag: SilenceForkPRErrorsFlag,
})

if err != nil {
Expand Down Expand Up @@ -921,8 +928,11 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig, v *viper.Viper) {
if c.RedisPort == 0 {
c.RedisPort = DefaultRedisPort
}
if c.TFDistribution == "" {
c.TFDistribution = DefaultTFDistribution
if c.TFDistribution != "" && c.DefaultTFDistribution == "" {
c.DefaultTFDistribution = c.TFDistribution
}
if c.DefaultTFDistribution == "" {
c.DefaultTFDistribution = DefaultTFDistribution
}
if c.TFDownloadURL == "" {
c.TFDownloadURL = DefaultTFDownloadURL
Expand Down Expand Up @@ -953,7 +963,7 @@ func (s *ServerCmd) validate(userConfig server.UserConfig) error {
return fmt.Errorf("invalid log level: must be one of %v", ValidLogLevels)
}

if userConfig.TFDistribution != TFDistributionTerraform && userConfig.TFDistribution != TFDistributionOpenTofu {
if userConfig.DefaultTFDistribution != TFDistributionTerraform && userConfig.DefaultTFDistribution != TFDistributionOpenTofu {
return fmt.Errorf("invalid tf distribution: expected one of %s or %s",
TFDistributionTerraform, TFDistributionOpenTofu)
}
Expand Down Expand Up @@ -1172,6 +1182,10 @@ func (s *ServerCmd) deprecationWarnings(userConfig *server.UserConfig) error {
// }
//

if userConfig.TFDistribution != "" {
deprecatedFlags = append(deprecatedFlags, TFDistributionFlag)
}

if len(deprecatedFlags) > 0 {
warning := "WARNING: "
if len(deprecatedFlags) == 1 {
Expand Down
1 change: 1 addition & 0 deletions cmd/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ var testFlags = map[string]interface{}{
CheckoutStrategyFlag: CheckoutStrategyMerge,
CheckoutDepthFlag: 0,
DataDirFlag: "/path",
DefaultTFDistributionFlag: "terraform",
DefaultTFVersionFlag: "v0.11.0",
DisableApplyAllFlag: true,
DisableMarkdownFoldingFlag: true,
Expand Down
17 changes: 11 additions & 6 deletions server/controllers/events/events_controller_e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
mock_policy "github.com/runatlantis/atlantis/server/core/runtime/policy/mocks"
"github.com/runatlantis/atlantis/server/core/terraform"
terraform_mocks "github.com/runatlantis/atlantis/server/core/terraform/mocks"
"github.com/runatlantis/atlantis/server/core/terraform/tfclient"
"github.com/runatlantis/atlantis/server/events"
"github.com/runatlantis/atlantis/server/events/command"
"github.com/runatlantis/atlantis/server/events/mocks"
Expand Down Expand Up @@ -1319,7 +1320,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
mockDownloader := terraform_mocks.NewMockDownloader()
distribution := terraform.NewDistributionTerraformWithDownloader(mockDownloader)

terraformClient, err := terraform.NewClient(logger, distribution, binDir, cacheDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", true, false, projectCmdOutputHandler)
terraformClient, err := tfclient.NewClient(logger, distribution, binDir, cacheDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", true, false, projectCmdOutputHandler)
Ok(t, err)
boltdb, err := db.New(dataDir)
Ok(t, err)
Expand All @@ -1346,6 +1347,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
}
}

defaultTFDistribution := terraformClient.DefaultDistribution()
defaultTFVersion := terraformClient.DefaultVersion()
locker := events.NewDefaultWorkingDirLocker()
parser := &config.ParserValidator{}
Expand Down Expand Up @@ -1429,7 +1431,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
terraformClient,
)

showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTFVersion)
showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTFDistribution, defaultTFVersion)

Ok(t, err)

Expand All @@ -1440,6 +1442,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
conftextExec.VersionCache = &LocalConftestCache{}

policyCheckRunner, err := runtime.NewPolicyCheckStepRunner(
defaultTFDistribution,
defaultTFVersion,
conftextExec,
)
Expand All @@ -1451,11 +1454,13 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
Locker: projectLocker,
LockURLGenerator: &mockLockURLGenerator{},
InitStepRunner: &runtime.InitStepRunner{
TerraformExecutor: terraformClient,
DefaultTFVersion: defaultTFVersion,
TerraformExecutor: terraformClient,
DefaultTFDistribution: defaultTFDistribution,
DefaultTFVersion: defaultTFVersion,
},
PlanStepRunner: runtime.NewPlanStepRunner(
terraformClient,
defaultTFDistribution,
defaultTFVersion,
statusUpdater,
asyncTfExec,
Expand All @@ -1465,8 +1470,8 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
ApplyStepRunner: &runtime.ApplyStepRunner{
TerraformExecutor: terraformClient,
},
ImportStepRunner: runtime.NewImportStepRunner(terraformClient, defaultTFVersion),
StateRmStepRunner: runtime.NewStateRmStepRunner(terraformClient, defaultTFVersion),
ImportStepRunner: runtime.NewImportStepRunner(terraformClient, defaultTFDistribution, defaultTFVersion),
StateRmStepRunner: runtime.NewStateRmStepRunner(terraformClient, defaultTFDistribution, defaultTFVersion),
RunStepRunner: &runtime.RunStepRunner{
TerraformExecutor: terraformClient,
DefaultTFVersion: defaultTFVersion,
Expand Down
25 changes: 25 additions & 0 deletions server/core/config/parser_validator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,31 @@ workflows:
},
},
},
{
description: "project field with terraform_distribution set to opentofu",
input: `
version: 3
projects:
- dir: .
workspace: myworkspace
terraform_distribution: opentofu
`,
exp: valid.RepoCfg{
Version: 3,
Projects: []valid.Project{
{
Dir: ".",
Workspace: "myworkspace",
TerraformDistribution: String("opentofu"),
Autoplan: valid.Autoplan{
WhenModified: raw.DefaultAutoPlanWhenModified,
Enabled: true,
},
},
},
Workflows: make(map[string]valid.Workflow),
},
},
{
description: "project dir with ..",
input: `
Expand Down
13 changes: 13 additions & 0 deletions server/core/config/raw/project.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type Project struct {
Dir *string `yaml:"dir,omitempty"`
Workspace *string `yaml:"workspace,omitempty"`
Workflow *string `yaml:"workflow,omitempty"`
TerraformDistribution *string `yaml:"terraform_distribution,omitempty"`
TerraformVersion *string `yaml:"terraform_version,omitempty"`
Autoplan *Autoplan `yaml:"autoplan,omitempty"`
PlanRequirements []string `yaml:"plan_requirements,omitempty"`
Expand Down Expand Up @@ -86,6 +87,7 @@ func (p Project) Validate() error {
validation.Field(&p.PlanRequirements, validation.By(validPlanReq)),
validation.Field(&p.ApplyRequirements, validation.By(validApplyReq)),
validation.Field(&p.ImportRequirements, validation.By(validImportReq)),
validation.Field(&p.TerraformDistribution, validation.By(validDistribution)),
validation.Field(&p.TerraformVersion, validation.By(VersionValidator)),
validation.Field(&p.DependsOn, validation.By(DependsOn)),
validation.Field(&p.Name, validation.By(validName)),
Expand Down Expand Up @@ -118,6 +120,9 @@ func (p Project) ToValid() valid.Project {
if p.TerraformVersion != nil {
v.TerraformVersion, _ = version.NewVersion(*p.TerraformVersion)
}
if p.TerraformDistribution != nil {
v.TerraformDistribution = p.TerraformDistribution
}
if p.Autoplan == nil {
v.Autoplan = DefaultAutoPlan()
} else {
Expand Down Expand Up @@ -202,3 +207,11 @@ func validImportReq(value interface{}) error {
}
return nil
}

func validDistribution(value interface{}) error {
distribution := value.(*string)
if distribution != nil && *distribution != "terraform" && *distribution != "opentofu" {
return fmt.Errorf("%q is not a valid terraform_distribution, only %q and %q are supported", *distribution, "terraform", "opentofu")
}
return nil
}
3 changes: 3 additions & 0 deletions server/core/config/valid/global_cfg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ type MergedProjectCfg struct {
AutoplanEnabled bool
AutoMergeDisabled bool
AutoMergeMethod string
TerraformDistribution *string
TerraformVersion *version.Version
RepoCfgVersion int
PolicySets PolicySets
Expand Down Expand Up @@ -412,6 +413,7 @@ func (g GlobalCfg) MergeProjectCfg(log logging.SimpleLogging, repoID string, pro
DependsOn: proj.DependsOn,
Name: proj.GetName(),
AutoplanEnabled: proj.Autoplan.Enabled,
TerraformDistribution: proj.TerraformDistribution,
TerraformVersion: proj.TerraformVersion,
RepoCfgVersion: rCfg.Version,
PolicySets: g.PolicySets,
Expand All @@ -438,6 +440,7 @@ func (g GlobalCfg) DefaultProjCfg(log logging.SimpleLogging, repoID string, repo
Workspace: workspace,
Name: "",
AutoplanEnabled: DefaultAutoPlanEnabled,
TerraformDistribution: nil,
TerraformVersion: nil,
PolicySets: g.PolicySets,
DeleteSourceBranchOnMerge: deleteSourceBranchOnMerge,
Expand Down
1 change: 1 addition & 0 deletions server/core/config/valid/repo_cfg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ type Project struct {
Workspace string
Name *string
WorkflowName *string
TerraformDistribution *string
TerraformVersion *version.Version
Autoplan Autoplan
PlanRequirements []string
Expand Down
25 changes: 18 additions & 7 deletions server/core/runtime/apply_step_runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,19 @@ import (
"github.com/pkg/errors"

version "github.com/hashicorp/go-version"
"github.com/runatlantis/atlantis/server/core/terraform"
"github.com/runatlantis/atlantis/server/events/command"
"github.com/runatlantis/atlantis/server/events/models"
"github.com/runatlantis/atlantis/server/utils"
)

// ApplyStepRunner runs `terraform apply`.
type ApplyStepRunner struct {
TerraformExecutor TerraformExec
DefaultTFVersion *version.Version
CommitStatusUpdater StatusUpdater
AsyncTFExec AsyncTFExec
TerraformExecutor TerraformExec
DefaultTFDistribution terraform.Distribution
DefaultTFVersion *version.Version
CommitStatusUpdater StatusUpdater
AsyncTFExec AsyncTFExec
}

func (a *ApplyStepRunner) Run(ctx command.ProjectContext, extraArgs []string, path string, envs map[string]string) (string, error) {
Expand All @@ -39,19 +41,27 @@ func (a *ApplyStepRunner) Run(ctx command.ProjectContext, extraArgs []string, pa

ctx.Log.Info("starting apply")
var out string
tfDistribution := a.DefaultTFDistribution
if ctx.TerraformDistribution != nil {
tfDistribution = terraform.NewDistribution(*ctx.TerraformDistribution)
}
tfVersion := a.DefaultTFVersion
if ctx.TerraformVersion != nil {
tfVersion = ctx.TerraformVersion
}

// TODO: Leverage PlanTypeStepRunnerDelegate here
if IsRemotePlan(contents) {
args := append(append([]string{"apply", "-input=false", "-no-color"}, extraArgs...), ctx.EscapedCommentArgs...)
out, err = a.runRemoteApply(ctx, args, path, planPath, ctx.TerraformVersion, envs)
out, err = a.runRemoteApply(ctx, args, path, planPath, tfDistribution, tfVersion, envs)
if err == nil {
out = a.cleanRemoteApplyOutput(out)
}
} else {
// NOTE: we need to quote the plan path because Bitbucket Server can
// have spaces in its repo owner names which is part of the path.
args := append(append(append([]string{"apply", "-input=false"}, extraArgs...), ctx.EscapedCommentArgs...), fmt.Sprintf("%q", planPath))
out, err = a.TerraformExecutor.RunCommandWithVersion(ctx, path, args, envs, ctx.TerraformVersion, ctx.Workspace)
out, err = a.TerraformExecutor.RunCommandWithVersion(ctx, path, args, envs, tfDistribution, tfVersion, ctx.Workspace)
}

// If the apply was successful, delete the plan.
Expand Down Expand Up @@ -115,6 +125,7 @@ func (a *ApplyStepRunner) runRemoteApply(
applyArgs []string,
path string,
absPlanPath string,
tfDistribution terraform.Distribution,
tfVersion *version.Version,
envs map[string]string) (string, error) {
// The planfile contents are needed to ensure that the plan didn't change
Expand All @@ -133,7 +144,7 @@ func (a *ApplyStepRunner) runRemoteApply(

// Start the async command execution.
ctx.Log.Debug("starting async tf remote operation")
inCh, outCh := a.AsyncTFExec.RunCommandAsync(ctx, filepath.Clean(path), applyArgs, envs, tfVersion, ctx.Workspace)
inCh, outCh := a.AsyncTFExec.RunCommandAsync(ctx, filepath.Clean(path), applyArgs, envs, tfDistribution, tfVersion, ctx.Workspace)
var lines []string
nextLineIsRunURL := false
var runURL string
Expand Down
Loading

0 comments on commit 20e0cc3

Please sign in to comment.