Skip to content

Commit

Permalink
Scrub secret vars
Browse files Browse the repository at this point in the history
- Scrub secret vars in RequiredVarNotFoundError
- Scrub secret vars in StateCheckVarsHash event
- Scrub secret vars in run results
  • Loading branch information
nielspardon committed Mar 7, 2024
1 parent deedeeb commit 638fbdb
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 3 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20240307-153622.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Support scrubbing secret vars
time: 2024-03-07T15:36:22.754627+01:00
custom:
Author: nielspardon
Issue: "7247"
24 changes: 23 additions & 1 deletion core/dbt/artifacts/schemas/run/v5/run.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import threading
from typing import Any, Optional, Iterable, Tuple, Sequence, Dict
import agate
import copy
from dataclasses import dataclass, field
from datetime import datetime


from dbt.constants import SECRET_ENV_PREFIX
from dbt.contracts.graph.nodes import CompiledNode
from dbt.artifacts.schemas.base import (
BaseArtifactMetadata,
Expand All @@ -20,6 +22,7 @@
ExecutionResult,
)
from dbt_common.clients.system import write_json
from dbt.exceptions import scrub_secrets


@dataclass
Expand Down Expand Up @@ -120,7 +123,26 @@ def from_execution_results(
dbt_schema_version=str(cls.dbt_schema_version),
generated_at=generated_at,
)
return cls(metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=args)

secret_vars = [

Check warning on line 127 in core/dbt/artifacts/schemas/run/v5/run.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/artifacts/schemas/run/v5/run.py#L127

Added line #L127 was not covered by tests
v for k, v in args["vars"].items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()
]

scrubbed_args = copy.deepcopy(args)

Check warning on line 131 in core/dbt/artifacts/schemas/run/v5/run.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/artifacts/schemas/run/v5/run.py#L131

Added line #L131 was not covered by tests

# scrub secrets in invocation command
scrubbed_args["invocation_command"] = scrub_secrets(

Check warning on line 134 in core/dbt/artifacts/schemas/run/v5/run.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/artifacts/schemas/run/v5/run.py#L134

Added line #L134 was not covered by tests
scrubbed_args["invocation_command"], secret_vars
)

# scrub secrets in vars dict
scrubbed_args["vars"] = {

Check warning on line 139 in core/dbt/artifacts/schemas/run/v5/run.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/artifacts/schemas/run/v5/run.py#L139

Added line #L139 was not covered by tests
k: scrub_secrets(v, secret_vars) for k, v in scrubbed_args["vars"].items()
}

return cls(

Check warning on line 143 in core/dbt/artifacts/schemas/run/v5/run.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/artifacts/schemas/run/v5/run.py#L143

Added line #L143 was not covered by tests
metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=scrubbed_args
)

@classmethod
def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:
Expand Down
7 changes: 6 additions & 1 deletion core/dbt/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

from dbt_common.dataclass_schema import ValidationError

from dbt.constants import SECRET_ENV_PREFIX


class ContractBreakingChangeError(DbtRuntimeError):
CODE = 10016
Expand Down Expand Up @@ -358,7 +360,10 @@ def get_message(self) -> str:
pretty_vars = json.dumps(dct, sort_keys=True, indent=4)

msg = f"Required var '{self.var_name}' not found in config:\nVars supplied to {node_name} = {pretty_vars}"
return msg
return scrub_secrets(msg, self.var_secrets())

def var_secrets(self) -> List[str]:
return [v for k, v in self.merged.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()]


class PackageNotFoundForMacroError(CompilationError):
Expand Down
7 changes: 6 additions & 1 deletion core/dbt/parser/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
MANIFEST_FILE_NAME,
PARTIAL_PARSE_FILE_NAME,
SEMANTIC_MANIFEST_FILE_NAME,
SECRET_ENV_PREFIX,
)
from dbt_common.helper_types import PathSet
from dbt_common.events.functions import fire_event, get_invocation_id, warn_or_error
Expand Down Expand Up @@ -113,6 +114,7 @@
TargetNotFoundError,
AmbiguousAliasError,
InvalidAccessTypeError,
scrub_secrets,
)
from dbt.parser.base import Parser
from dbt.parser.analysis import AnalysisParser
Expand Down Expand Up @@ -944,6 +946,9 @@ def build_manifest_state_check(self):
# of env_vars, that would need to change.
# We are using the parsed cli_vars instead of config.args.vars, in order
# to sort them and avoid reparsing because of ordering issues.
secret_vars = [

Check warning on line 949 in core/dbt/parser/manifest.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/parser/manifest.py#L949

Added line #L949 was not covered by tests
v for k, v in config.cli_vars.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()
]
stringified_cli_vars = pprint.pformat(config.cli_vars)
vars_hash = FileHash.from_contents(
"\x00".join(
Expand All @@ -958,7 +963,7 @@ def build_manifest_state_check(self):
fire_event(
StateCheckVarsHash(
checksum=vars_hash.checksum,
vars=stringified_cli_vars,
vars=scrub_secrets(stringified_cli_vars, secret_vars),
profile=config.args.profile,
target=config.args.target,
version=__version__,
Expand Down
35 changes: 35 additions & 0 deletions tests/functional/context_methods/test_cli_vars.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,3 +206,38 @@ def test_vars_in_selectors(self, project):
# Var in cli_vars works
results = run_dbt(["run", "--vars", "snapshot_target: dev"])
assert len(results) == 1


models_scrubbing__schema_yml = """
version: 2
models:
- name: simple_model
columns:
- name: simple
data_tests:
- accepted_values:
values:
- abc
"""

models_scrubbing__simple_model_sql = """
select
'{{ var("DBT_ENV_SECRET_simple") }}'::varchar as simple
"""


class TestCLIVarsScrubbing:
@pytest.fixture(scope="class")
def models(self):
return {
"schema.yml": models_scrubbing__schema_yml,
"simple_model.sql": models_scrubbing__simple_model_sql,
}

def test__run_results_scrubbing(self, project):
results = run_dbt(["run", "--vars", "{DBT_ENV_SECRET_simple: abc, unused: def}"])
assert len(results) == 1
results = run_dbt(["test", "--vars", "{DBT_ENV_SECRET_simple: abc, unused: def}"])
assert len(results) == 1
run_results = get_artifact(project.project_root, "target", "run_results.json")
assert run_results["args"]["vars"] == {"DBT_ENV_SECRET_simple": "*****", "unused": "def"}

0 comments on commit 638fbdb

Please sign in to comment.