diff --git a/.changes/unreleased/Features-20240307-153622.yaml b/.changes/unreleased/Features-20240307-153622.yaml new file mode 100644 index 00000000000..80886a82c9b --- /dev/null +++ b/.changes/unreleased/Features-20240307-153622.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support scrubbing secret vars +time: 2024-03-07T15:36:22.754627+01:00 +custom: + Author: nielspardon + Issue: "7247" diff --git a/core/dbt/artifacts/schemas/run/v5/run.py b/core/dbt/artifacts/schemas/run/v5/run.py index eb731b71b5d..78998ff3d5b 100644 --- a/core/dbt/artifacts/schemas/run/v5/run.py +++ b/core/dbt/artifacts/schemas/run/v5/run.py @@ -1,10 +1,12 @@ import threading from typing import Any, Optional, Iterable, Tuple, Sequence, Dict import agate +import copy from dataclasses import dataclass, field from datetime import datetime +from dbt.constants import SECRET_ENV_PREFIX from dbt.contracts.graph.nodes import CompiledNode from dbt.artifacts.schemas.base import ( BaseArtifactMetadata, @@ -20,6 +22,7 @@ ExecutionResult, ) from dbt_common.clients.system import write_json +from dbt.exceptions import scrub_secrets @dataclass @@ -120,7 +123,26 @@ def from_execution_results( dbt_schema_version=str(cls.dbt_schema_version), generated_at=generated_at, ) - return cls(metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=args) + + secret_vars = [ + v for k, v in args["vars"].items() if k.startswith(SECRET_ENV_PREFIX) and v.strip() + ] + + scrubbed_args = copy.deepcopy(args) + + # scrub secrets in invocation command + scrubbed_args["invocation_command"] = scrub_secrets( + scrubbed_args["invocation_command"], secret_vars + ) + + # scrub secrets in vars dict + scrubbed_args["vars"] = { + k: scrub_secrets(v, secret_vars) for k, v in scrubbed_args["vars"].items() + } + + return cls( + metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=scrubbed_args + ) @classmethod def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]: diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 25e4c055f14..4ebfa842832 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -18,6 +18,8 @@ from dbt_common.dataclass_schema import ValidationError +from dbt.constants import SECRET_ENV_PREFIX + class ContractBreakingChangeError(DbtRuntimeError): CODE = 10016 @@ -358,7 +360,10 @@ def get_message(self) -> str: pretty_vars = json.dumps(dct, sort_keys=True, indent=4) msg = f"Required var '{self.var_name}' not found in config:\nVars supplied to {node_name} = {pretty_vars}" - return msg + return scrub_secrets(msg, self.var_secrets()) + + def var_secrets(self) -> List[str]: + return [v for k, v in self.merged.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()] class PackageNotFoundForMacroError(CompilationError): diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index eb6ff5b5702..59dde6b51bd 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -43,6 +43,7 @@ MANIFEST_FILE_NAME, PARTIAL_PARSE_FILE_NAME, SEMANTIC_MANIFEST_FILE_NAME, + SECRET_ENV_PREFIX, ) from dbt_common.helper_types import PathSet from dbt_common.events.functions import fire_event, get_invocation_id, warn_or_error @@ -113,6 +114,7 @@ TargetNotFoundError, AmbiguousAliasError, InvalidAccessTypeError, + scrub_secrets, ) from dbt.parser.base import Parser from dbt.parser.analysis import AnalysisParser @@ -944,6 +946,9 @@ def build_manifest_state_check(self): # of env_vars, that would need to change. # We are using the parsed cli_vars instead of config.args.vars, in order # to sort them and avoid reparsing because of ordering issues. + secret_vars = [ + v for k, v in config.cli_vars.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip() + ] stringified_cli_vars = pprint.pformat(config.cli_vars) vars_hash = FileHash.from_contents( "\x00".join( @@ -958,7 +963,7 @@ def build_manifest_state_check(self): fire_event( StateCheckVarsHash( checksum=vars_hash.checksum, - vars=stringified_cli_vars, + vars=scrub_secrets(stringified_cli_vars, secret_vars), profile=config.args.profile, target=config.args.target, version=__version__, diff --git a/tests/functional/context_methods/test_cli_vars.py b/tests/functional/context_methods/test_cli_vars.py index d3d5dfc8197..c40c577e50a 100644 --- a/tests/functional/context_methods/test_cli_vars.py +++ b/tests/functional/context_methods/test_cli_vars.py @@ -206,3 +206,38 @@ def test_vars_in_selectors(self, project): # Var in cli_vars works results = run_dbt(["run", "--vars", "snapshot_target: dev"]) assert len(results) == 1 + + +models_scrubbing__schema_yml = """ +version: 2 +models: +- name: simple_model + columns: + - name: simple + data_tests: + - accepted_values: + values: + - abc +""" + +models_scrubbing__simple_model_sql = """ +select + '{{ var("DBT_ENV_SECRET_simple") }}'::varchar as simple +""" + + +class TestCLIVarsScrubbing: + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": models_scrubbing__schema_yml, + "simple_model.sql": models_scrubbing__simple_model_sql, + } + + def test__run_results_scrubbing(self, project): + results = run_dbt(["run", "--vars", "{DBT_ENV_SECRET_simple: abc, unused: def}"]) + assert len(results) == 1 + results = run_dbt(["test", "--vars", "{DBT_ENV_SECRET_simple: abc, unused: def}"]) + assert len(results) == 1 + run_results = get_artifact(project.project_root, "target", "run_results.json") + assert run_results["args"]["vars"] == {"DBT_ENV_SECRET_simple": "*****", "unused": "def"}