diff --git a/.changes/unreleased/Breaking Changes-20221205-141937.yaml b/.changes/unreleased/Breaking Changes-20221205-141937.yaml new file mode 100644 index 00000000000..be840b20a99 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20221205-141937.yaml @@ -0,0 +1,9 @@ +kind: Breaking Changes +body: Cleaned up exceptions to directly raise in code. Removed use of all exception + functions in the code base and marked them all as deprecated to be removed next + minor release. +time: 2022-12-05T14:19:37.863032-06:00 +custom: + Author: emmyoop + Issue: "6339" + PR: "6347" diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index bbac18cb16b..64ebbeac5dd 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -22,13 +22,20 @@ import pytz from dbt.exceptions import ( - raise_database_error, - raise_compiler_error, - invalid_type_error, - get_relation_returned_multiple_results, InternalException, + InvalidMacroArgType, + InvalidMacroResult, + InvalidQuoteConfigType, NotImplementedException, + NullRelationCacheAttempted, + NullRelationDropAttempted, + RelationReturnedMultipleResults, + RenameToNoneAttempted, RuntimeException, + SnapshotTargetIncomplete, + SnapshotTargetNotSnapshotTable, + UnexpectedNull, + UnexpectedNonTimestamp, ) from dbt.adapters.protocol import ( @@ -97,18 +104,10 @@ def _utc(dt: Optional[datetime], source: BaseRelation, field_name: str) -> datet assume the datetime is already for UTC and add the timezone. """ if dt is None: - raise raise_database_error( - "Expected a non-null value when querying field '{}' of table " - " {} but received value 'null' instead".format(field_name, source) - ) + raise UnexpectedNull(field_name, source) elif not hasattr(dt, "tzinfo"): - raise raise_database_error( - "Expected a timestamp value when querying field '{}' of table " - "{} but received value of type '{}' instead".format( - field_name, source, type(dt).__name__ - ) - ) + raise UnexpectedNonTimestamp(field_name, source, dt) elif dt.tzinfo: return dt.astimezone(pytz.UTC) @@ -434,7 +433,7 @@ def cache_added(self, relation: Optional[BaseRelation]) -> str: """Cache a new relation in dbt. It will show up in `list relations`.""" if relation is None: name = self.nice_connection_name() - raise_compiler_error("Attempted to cache a null relation for {}".format(name)) + raise NullRelationCacheAttempted(name) self.cache.add(relation) # so jinja doesn't render things return "" @@ -446,7 +445,7 @@ def cache_dropped(self, relation: Optional[BaseRelation]) -> str: """ if relation is None: name = self.nice_connection_name() - raise_compiler_error("Attempted to drop a null relation for {}".format(name)) + raise NullRelationDropAttempted(name) self.cache.drop(relation) return "" @@ -463,9 +462,7 @@ def cache_renamed( name = self.nice_connection_name() src_name = _relation_name(from_relation) dst_name = _relation_name(to_relation) - raise_compiler_error( - "Attempted to rename {} to {} for {}".format(src_name, dst_name, name) - ) + raise RenameToNoneAttempted(src_name, dst_name, name) self.cache.rename(from_relation, to_relation) return "" @@ -615,7 +612,7 @@ def get_missing_columns( to_relation. """ if not isinstance(from_relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="get_missing_columns", arg_name="from_relation", got_value=from_relation, @@ -623,7 +620,7 @@ def get_missing_columns( ) if not isinstance(to_relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="get_missing_columns", arg_name="to_relation", got_value=to_relation, @@ -648,7 +645,7 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: incorrect. """ if not isinstance(relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="valid_snapshot_target", arg_name="relation", got_value=relation, @@ -669,24 +666,16 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: if missing: if extra: - msg = ( - 'Snapshot target has ("{}") but not ("{}") - is it an ' - "unmigrated previous version archive?".format( - '", "'.join(extra), '", "'.join(missing) - ) - ) + raise SnapshotTargetIncomplete(extra, missing) else: - msg = 'Snapshot target is not a snapshot table (missing "{}")'.format( - '", "'.join(missing) - ) - raise_compiler_error(msg) + raise SnapshotTargetNotSnapshotTable(missing) @available.parse_none def expand_target_column_types( self, from_relation: BaseRelation, to_relation: BaseRelation ) -> None: if not isinstance(from_relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="expand_target_column_types", arg_name="from_relation", got_value=from_relation, @@ -694,7 +683,7 @@ def expand_target_column_types( ) if not isinstance(to_relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="expand_target_column_types", arg_name="to_relation", got_value=to_relation, @@ -776,7 +765,7 @@ def get_relation(self, database: str, schema: str, identifier: str) -> Optional[ "schema": schema, "database": database, } - get_relation_returned_multiple_results(kwargs, matches) + raise RelationReturnedMultipleResults(kwargs, matches) elif matches: return matches[0] @@ -840,10 +829,7 @@ def quote_seed_column(self, column: str, quote_config: Optional[bool]) -> str: elif quote_config is None: pass else: - raise_compiler_error( - f'The seed configuration value of "quote_columns" has an ' - f"invalid type {type(quote_config)}" - ) + raise InvalidQuoteConfigType(quote_config) if quote_columns: return self.quote(column) @@ -1093,11 +1079,7 @@ def calculate_freshness( # now we have a 1-row table of the maximum `loaded_at_field` value and # the current time according to the db. if len(table) != 1 or len(table[0]) != 2: - raise_compiler_error( - 'Got an invalid result from "{}" macro: {}'.format( - FRESHNESS_MACRO_NAME, [tuple(r) for r in table] - ) - ) + raise InvalidMacroResult(FRESHNESS_MACRO_NAME, table) if table[0][0] is None: # no records in the table, so really the max_loaded_at was # infinitely long ago. Just call it 0:00 January 1 year UTC diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index 0461990c92d..5bc0c56b264 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -11,7 +11,7 @@ Policy, Path, ) -from dbt.exceptions import InternalException +from dbt.exceptions import ApproximateMatch, InternalException, MultipleDatabasesNotAllowed from dbt.node_types import NodeType from dbt.utils import filter_null_values, deep_merge, classproperty @@ -100,7 +100,7 @@ def matches( if approximate_match and not exact_match: target = self.create(database=database, schema=schema, identifier=identifier) - dbt.exceptions.approximate_relation_match(target, self) + raise ApproximateMatch(target, self) return exact_match @@ -438,7 +438,7 @@ def flatten(self, allow_multiple_databases: bool = False): if not allow_multiple_databases: seen = {r.database.lower() for r in self if r.database} if len(seen) > 1: - dbt.exceptions.raise_compiler_error(str(seen)) + raise MultipleDatabasesNotAllowed(seen) for information_schema_name, schema in self.search(): path = {"database": information_schema_name.database, "schema": schema} diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 6c60039f262..90c4cab27fb 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -1,4 +1,3 @@ -import re import threading from copy import deepcopy from typing import Any, Dict, Iterable, List, Optional, Set, Tuple @@ -9,7 +8,13 @@ _make_msg_from_ref_key, _ReferenceKey, ) -import dbt.exceptions +from dbt.exceptions import ( + DependentLinkNotCached, + NewNameAlreadyInCache, + NoneRelationFound, + ReferencedLinkNotCached, + TruncatedModelNameCausedCollision, +) from dbt.events.functions import fire_event, fire_event_if from dbt.events.types import ( AddLink, @@ -150,11 +155,7 @@ def rename_key(self, old_key, new_key): :raises InternalError: If the new key already exists. """ if new_key in self.referenced_by: - dbt.exceptions.raise_cache_inconsistent( - 'in rename of "{}" -> "{}", new name is in the cache already'.format( - old_key, new_key - ) - ) + raise NewNameAlreadyInCache(old_key, new_key) if old_key not in self.referenced_by: return @@ -270,15 +271,11 @@ def _add_link(self, referenced_key, dependent_key): if referenced is None: return if referenced is None: - dbt.exceptions.raise_cache_inconsistent( - "in add_link, referenced link key {} not in cache!".format(referenced_key) - ) + raise ReferencedLinkNotCached(referenced_key) dependent = self.relations.get(dependent_key) if dependent is None: - dbt.exceptions.raise_cache_inconsistent( - "in add_link, dependent link key {} not in cache!".format(dependent_key) - ) + raise DependentLinkNotCached(dependent_key) assert dependent is not None # we just raised! @@ -430,24 +427,7 @@ def _check_rename_constraints(self, old_key, new_key): if new_key in self.relations: # Tell user when collision caused by model names truncated during # materialization. - match = re.search("__dbt_backup|__dbt_tmp$", new_key.identifier) - if match: - truncated_model_name_prefix = new_key.identifier[: match.start()] - message_addendum = ( - "\n\nName collisions can occur when the length of two " - "models' names approach your database's builtin limit. " - "Try restructuring your project such that no two models " - "share the prefix '{}'.".format(truncated_model_name_prefix) - + " Then, clean your warehouse of any removed models." - ) - else: - message_addendum = "" - - dbt.exceptions.raise_cache_inconsistent( - "in rename, new key {} already in cache: {}{}".format( - new_key, list(self.relations.keys()), message_addendum - ) - ) + raise TruncatedModelNameCausedCollision(new_key, self.relations) if old_key not in self.relations: fire_event(TemporaryRelation(key=_make_msg_from_ref_key(old_key))) @@ -505,9 +485,7 @@ def get_relations(self, database: Optional[str], schema: Optional[str]) -> List[ ] if None in results: - dbt.exceptions.raise_cache_inconsistent( - "in get_relations, a None relation was found in the cache!" - ) + raise NoneRelationFound() return results def clear(self): diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index 20241d9e53d..4606b046f54 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -1,9 +1,8 @@ import agate from typing import Any, Optional, Tuple, Type, List -import dbt.clients.agate_helper from dbt.contracts.connection import Connection -import dbt.exceptions +from dbt.exceptions import RelationTypeNull from dbt.adapters.base import BaseAdapter, available from dbt.adapters.cache import _make_ref_key_msg from dbt.adapters.sql import SQLConnectionManager @@ -132,9 +131,7 @@ def alter_column_type(self, relation, column_name, new_column_type) -> None: def drop_relation(self, relation): if relation.type is None: - dbt.exceptions.raise_compiler_error( - "Tried to drop relation {}, but its type is null.".format(relation) - ) + raise RelationTypeNull(relation) self.cache_dropped(relation) self.execute_macro(DROP_RELATION_MACRO_NAME, kwargs={"relation": relation}) diff --git a/core/dbt/clients/_jinja_blocks.py b/core/dbt/clients/_jinja_blocks.py index c1ef31acf44..fa74a317649 100644 --- a/core/dbt/clients/_jinja_blocks.py +++ b/core/dbt/clients/_jinja_blocks.py @@ -1,7 +1,15 @@ import re from collections import namedtuple -import dbt.exceptions +from dbt.exceptions import ( + BlockDefinitionNotAtTop, + InternalException, + MissingCloseTag, + MissingControlFlowStartTag, + NestedTags, + UnexpectedControlFlowEndTag, + UnexpectedMacroEOF, +) def regex(pat): @@ -139,10 +147,7 @@ def _first_match(self, *patterns, **kwargs): def _expect_match(self, expected_name, *patterns, **kwargs): match = self._first_match(*patterns, **kwargs) if match is None: - msg = 'unexpected EOF, expected {}, got "{}"'.format( - expected_name, self.data[self.pos :] - ) - dbt.exceptions.raise_compiler_error(msg) + raise UnexpectedMacroEOF(expected_name, self.data[self.pos :]) return match def handle_expr(self, match): @@ -256,7 +261,7 @@ def find_tags(self): elif block_type_name is not None: yield self.handle_tag(match) else: - raise dbt.exceptions.InternalException( + raise InternalException( "Invalid regex match in next_block, expected block start, " "expr start, or comment start" ) @@ -265,13 +270,6 @@ def __iter__(self): return self.find_tags() -duplicate_tags = ( - "Got nested tags: {outer.block_type_name} (started at {outer.start}) did " - "not have a matching {{% end{outer.block_type_name} %}} before a " - "subsequent {inner.block_type_name} was found (started at {inner.start})" -) - - _CONTROL_FLOW_TAGS = { "if": "endif", "for": "endfor", @@ -319,33 +317,16 @@ def find_blocks(self, allowed_blocks=None, collect_raw_data=True): found = self.stack.pop() else: expected = _CONTROL_FLOW_END_TAGS[tag.block_type_name] - dbt.exceptions.raise_compiler_error( - ( - "Got an unexpected control flow end tag, got {} but " - "never saw a preceeding {} (@ {})" - ).format(tag.block_type_name, expected, self.tag_parser.linepos(tag.start)) - ) + raise UnexpectedControlFlowEndTag(tag, expected, self.tag_parser) expected = _CONTROL_FLOW_TAGS[found] if expected != tag.block_type_name: - dbt.exceptions.raise_compiler_error( - ( - "Got an unexpected control flow end tag, got {} but " - "expected {} next (@ {})" - ).format(tag.block_type_name, expected, self.tag_parser.linepos(tag.start)) - ) + raise MissingControlFlowStartTag(tag, expected, self.tag_parser) if tag.block_type_name in allowed_blocks: if self.stack: - dbt.exceptions.raise_compiler_error( - ( - "Got a block definition inside control flow at {}. " - "All dbt block definitions must be at the top level" - ).format(self.tag_parser.linepos(tag.start)) - ) + raise BlockDefinitionNotAtTop(self.tag_parser, tag.start) if self.current is not None: - dbt.exceptions.raise_compiler_error( - duplicate_tags.format(outer=self.current, inner=tag) - ) + raise NestedTags(outer=self.current, inner=tag) if collect_raw_data: raw_data = self.data[self.last_position : tag.start] self.last_position = tag.start @@ -366,11 +347,7 @@ def find_blocks(self, allowed_blocks=None, collect_raw_data=True): if self.current: linecount = self.data[: self.current.end].count("\n") + 1 - dbt.exceptions.raise_compiler_error( - ("Reached EOF without finding a close tag for {} (searched from line {})").format( - self.current.block_type_name, linecount - ) - ) + raise MissingCloseTag(self.current.block_type_name, linecount) if collect_raw_data: raw_data = self.data[self.last_position :] diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py index 9eaa93203e0..4ddbb1969ee 100644 --- a/core/dbt/clients/git.py +++ b/core/dbt/clients/git.py @@ -14,10 +14,10 @@ ) from dbt.exceptions import ( CommandResultError, + GitCheckoutError, + GitCloningError, + GitCloningProblem, RuntimeException, - bad_package_spec, - raise_git_cloning_error, - raise_git_cloning_problem, ) from packaging import version @@ -27,16 +27,6 @@ def _is_commit(revision: str) -> bool: return bool(re.match(r"\b[0-9a-f]{40}\b", revision)) -def _raise_git_cloning_error(repo, revision, error): - stderr = error.stderr.strip() - if "usage: git" in stderr: - stderr = stderr.split("\nusage: git")[0] - if re.match("fatal: destination path '(.+)' already exists", stderr): - raise_git_cloning_error(error) - - bad_package_spec(repo, revision, stderr) - - def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None): has_revision = revision is not None is_commit = _is_commit(revision or "") @@ -64,7 +54,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec try: result = run_cmd(cwd, clone_cmd, env={"LC_ALL": "C"}) except CommandResultError as exc: - _raise_git_cloning_error(repo, revision, exc) + raise GitCloningError(repo, revision, exc) if subdirectory: cwd_subdir = os.path.join(cwd, dirname or "") @@ -72,7 +62,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec try: run_cmd(cwd_subdir, clone_cmd_subdir) except CommandResultError as exc: - _raise_git_cloning_error(repo, revision, exc) + raise GitCloningError(repo, revision, exc) if remove_git_dir: rmdir(os.path.join(dirname, ".git")) @@ -115,8 +105,7 @@ def checkout(cwd, repo, revision=None): try: return _checkout(cwd, repo, revision) except CommandResultError as exc: - stderr = exc.stderr.strip() - bad_package_spec(repo, revision, stderr) + raise GitCheckoutError(repo=repo, revision=revision, error=exc) def get_current_sha(cwd): @@ -145,7 +134,7 @@ def clone_and_checkout( err = exc.stderr exists = re.match("fatal: destination path '(.+)' already exists", err) if not exists: - raise_git_cloning_problem(repo) + raise GitCloningProblem(repo) directory = None start_sha = None diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index ac04bb86cb4..c1b8865e33e 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -28,12 +28,16 @@ from dbt.contracts.graph.nodes import GenericTestNode from dbt.exceptions import ( - InternalException, - raise_compiler_error, + CaughtMacroException, + CaughtMacroExceptionWithNode, CompilationException, - invalid_materialization_argument, - MacroReturn, + InternalException, + InvalidMaterializationArg, JinjaRenderingException, + MacroReturn, + MaterializtionMacroNotUsed, + NoSupportedLanguagesFound, + UndefinedCompilation, UndefinedMacroException, ) from dbt import flags @@ -237,7 +241,7 @@ def exception_handler(self) -> Iterator[None]: try: yield except (TypeError, jinja2.exceptions.TemplateRuntimeError) as e: - raise_compiler_error(str(e)) + raise CaughtMacroException(e) def call_macro(self, *args, **kwargs): # called from __call__ methods @@ -296,7 +300,7 @@ def exception_handler(self) -> Iterator[None]: try: yield except (TypeError, jinja2.exceptions.TemplateRuntimeError) as e: - raise_compiler_error(str(e), self.macro) + raise CaughtMacroExceptionWithNode(exc=e, node=self.macro) except CompilationException as e: e.stack.append(self.macro) raise e @@ -376,7 +380,7 @@ def parse(self, parser): node.defaults.append(languages) else: - invalid_materialization_argument(materialization_name, target.name) + raise InvalidMaterializationArg(materialization_name, target.name) if SUPPORTED_LANG_ARG not in node.args: node.args.append(SUPPORTED_LANG_ARG) @@ -451,7 +455,7 @@ def __call__(self, *args, **kwargs): return self def __reduce__(self): - raise_compiler_error(f"{self.name} is undefined", node=node) + raise UndefinedCompilation(name=self.name, node=node) return Undefined @@ -651,13 +655,13 @@ def _convert_function(value: Any, keypath: Tuple[Union[str, int], ...]) -> Any: def get_supported_languages(node: jinja2.nodes.Macro) -> List[ModelLanguage]: if "materialization" not in node.name: - raise_compiler_error("Only materialization macros can be used with this function") + raise MaterializtionMacroNotUsed(node=node) no_kwargs = not node.defaults no_langs_found = SUPPORTED_LANG_ARG not in node.args if no_kwargs or no_langs_found: - raise_compiler_error(f"No supported_languages found in materialization macro {node.name}") + raise NoSupportedLanguagesFound(node=node) lang_idx = node.args.index(SUPPORTED_LANG_ARG) # indexing defaults from the end diff --git a/core/dbt/clients/jinja_static.py b/core/dbt/clients/jinja_static.py index 337a25eadda..d71211cea6e 100644 --- a/core/dbt/clients/jinja_static.py +++ b/core/dbt/clients/jinja_static.py @@ -1,6 +1,6 @@ import jinja2 from dbt.clients.jinja import get_environment -from dbt.exceptions import raise_compiler_error +from dbt.exceptions import MacroNamespaceNotString, MacroNameNotString def statically_extract_macro_calls(string, ctx, db_wrapper=None): @@ -117,20 +117,14 @@ def statically_parse_adapter_dispatch(func_call, ctx, db_wrapper): func_name = kwarg.value.value possible_macro_calls.append(func_name) else: - raise_compiler_error( - f"The macro_name parameter ({kwarg.value.value}) " - "to adapter.dispatch was not a string" - ) + raise MacroNameNotString(kwarg_value=kwarg.value.value) elif kwarg.key == "macro_namespace": # This will remain to enable static resolution kwarg_type = type(kwarg.value).__name__ if kwarg_type == "Const": macro_namespace = kwarg.value.value else: - raise_compiler_error( - "The macro_namespace parameter to adapter.dispatch " - f"is a {kwarg_type}, not a string" - ) + raise MacroNamespaceNotString(kwarg_type) # positional arguments if packages_arg: diff --git a/core/dbt/clients/system.py b/core/dbt/clients/system.py index b1cd1b5a074..b776e91b1d0 100644 --- a/core/dbt/clients/system.py +++ b/core/dbt/clients/system.py @@ -144,7 +144,8 @@ def make_symlink(source: str, link_path: str) -> None: Create a symlink at `link_path` referring to `source`. """ if not supports_symlinks(): - dbt.exceptions.system_error("create a symbolic link") + # TODO: why not import these at top? + raise dbt.exceptions.SymbolicLinkError() os.symlink(source, link_path) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index fcf98b4e914..4ae78fd3485 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -21,7 +21,7 @@ SeedNode, ) from dbt.exceptions import ( - dependency_not_found, + GraphDependencyNotFound, InternalException, RuntimeException, ) @@ -399,7 +399,7 @@ def link_node(self, linker: Linker, node: GraphMemberNode, manifest: Manifest): elif dependency in manifest.metrics: linker.dependency(node.unique_id, (manifest.metrics[dependency].unique_id)) else: - dependency_not_found(node, dependency) + raise GraphDependencyNotFound(node, dependency) def link_graph(self, linker: Linker, manifest: Manifest, add_test_edges: bool = False): for source in manifest.sources.values(): diff --git a/core/dbt/config/profile.py b/core/dbt/config/profile.py index 39679baa109..e8bf85dbd27 100644 --- a/core/dbt/config/profile.py +++ b/core/dbt/config/profile.py @@ -9,12 +9,14 @@ from dbt.clients.yaml_helper import load_yaml_text from dbt.contracts.connection import Credentials, HasCredentials from dbt.contracts.project import ProfileConfig, UserConfig -from dbt.exceptions import CompilationException -from dbt.exceptions import DbtProfileError -from dbt.exceptions import DbtProjectError -from dbt.exceptions import ValidationException -from dbt.exceptions import RuntimeException -from dbt.exceptions import validator_error_message +from dbt.exceptions import ( + CompilationException, + DbtProfileError, + DbtProjectError, + ValidationException, + RuntimeException, + ProfileConfigInvalid, +) from dbt.events.types import MissingProfileTarget from dbt.events.functions import fire_event from dbt.utils import coerce_dict_str @@ -156,7 +158,7 @@ def validate(self): dct = self.to_profile_info(serialize_credentials=True) ProfileConfig.validate(dct) except ValidationError as exc: - raise DbtProfileError(validator_error_message(exc)) from exc + raise ProfileConfigInvalid(exc) from exc @staticmethod def _credentials_from_profile( diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 9521dd29882..69c6b79866c 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -16,19 +16,19 @@ import os from dbt import flags, deprecations -from dbt.clients.system import resolve_path_from_base -from dbt.clients.system import path_exists -from dbt.clients.system import load_file_contents +from dbt.clients.system import path_exists, resolve_path_from_base, load_file_contents from dbt.clients.yaml_helper import load_yaml_text from dbt.contracts.connection import QueryComment -from dbt.exceptions import DbtProjectError -from dbt.exceptions import SemverException -from dbt.exceptions import validator_error_message -from dbt.exceptions import RuntimeException +from dbt.exceptions import ( + DbtProjectError, + SemverException, + ProjectContractBroken, + ProjectContractInvalid, + RuntimeException, +) from dbt.graph import SelectionSpec from dbt.helper_types import NoValue -from dbt.semver import VersionSpecifier -from dbt.semver import versions_compatible +from dbt.semver import VersionSpecifier, versions_compatible from dbt.version import get_installed_version from dbt.utils import MultiDict from dbt.node_types import NodeType @@ -325,7 +325,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": ProjectContract.validate(rendered.project_dict) cfg = ProjectContract.from_dict(rendered.project_dict) except ValidationError as e: - raise DbtProjectError(validator_error_message(e)) from e + raise ProjectContractInvalid(e) from e # name/version are required in the Project definition, so we can assume # they are present name = cfg.name @@ -642,7 +642,7 @@ def validate(self): try: ProjectContract.validate(self.to_project_config()) except ValidationError as e: - raise DbtProjectError(validator_error_message(e)) from e + raise ProjectContractBroken(e) from e @classmethod def partial_load(cls, project_root: str, *, verify_version: bool = False) -> PartialProject: diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index 236baf497a6..8b1b30f383b 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -25,10 +25,11 @@ from dbt.contracts.relation import ComponentName from dbt.dataclass_schema import ValidationError from dbt.exceptions import ( + ConfigContractBroken, DbtProjectError, + NonUniquePackageName, RuntimeException, - raise_compiler_error, - validator_error_message, + UninstalledPackagesFound, ) from dbt.events.functions import warn_or_error from dbt.events.types import UnusedResourceConfigPath @@ -186,7 +187,7 @@ def validate(self): try: Configuration.validate(self.serialize()) except ValidationError as e: - raise DbtProjectError(validator_error_message(e)) from e + raise ConfigContractBroken(e) from e @classmethod def _get_rendered_profile( @@ -352,22 +353,15 @@ def load_dependencies(self, base_only=False) -> Mapping[str, "RuntimeConfig"]: count_packages_specified = len(self.packages.packages) # type: ignore count_packages_installed = len(tuple(self._get_project_directories())) if count_packages_specified > count_packages_installed: - raise_compiler_error( - f"dbt found {count_packages_specified} package(s) " - f"specified in packages.yml, but only " - f"{count_packages_installed} package(s) installed " - f'in {self.packages_install_path}. Run "dbt deps" to ' - f"install package dependencies." + raise UninstalledPackagesFound( + count_packages_specified, + count_packages_installed, + self.packages_install_path, ) project_paths = itertools.chain(internal_packages, self._get_project_directories()) for project_name, project in self.load_projects(project_paths): if project_name in all_projects: - raise_compiler_error( - f"dbt found more than one package with the name " - f'"{project_name}" included in this project. Package ' - f"names must be unique in a project. Please rename " - f"one of these packages." - ) + raise NonUniquePackageName(project_name) all_projects[project_name] = project self.dependencies = all_projects return self.dependencies diff --git a/core/dbt/config/utils.py b/core/dbt/config/utils.py index 728e558ebbd..921626ba088 100644 --- a/core/dbt/config/utils.py +++ b/core/dbt/config/utils.py @@ -9,7 +9,7 @@ from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer from dbt.events.functions import fire_event from dbt.events.types import InvalidVarsYAML -from dbt.exceptions import ValidationException, raise_compiler_error +from dbt.exceptions import ValidationException, VarsArgNotYamlDict def parse_cli_vars(var_string: str) -> Dict[str, Any]: @@ -19,11 +19,7 @@ def parse_cli_vars(var_string: str) -> Dict[str, Any]: if var_type is dict: return cli_vars else: - type_name = var_type.__name__ - raise_compiler_error( - "The --vars argument must be a YAML dictionary, but was " - "of type '{}'".format(type_name) - ) + raise VarsArgNotYamlDict(var_type) except ValidationException: fire_event(InvalidVarsYAML()) raise diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index e57c3edac56..59984cb96ab 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -10,11 +10,12 @@ from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER from dbt.contracts.graph.nodes import Resource from dbt.exceptions import ( - CompilationException, + DisallowSecretEnvVar, + EnvVarMissing, MacroReturn, - raise_compiler_error, - raise_parsing_error, - disallow_secret_env_var, + RequiredVarNotFound, + SetStrictWrongType, + ZipStrictWrongType, ) from dbt.events.functions import fire_event, get_invocation_id from dbt.events.types import JinjaLogInfo, JinjaLogDebug @@ -128,7 +129,6 @@ def __new__(mcls, name, bases, dct): class Var: - UndefinedVarError = "Required var '{}' not found in config:\nVars supplied to {} = {}" _VAR_NOTSET = object() def __init__( @@ -153,10 +153,7 @@ def node_name(self): return "" def get_missing_var(self, var_name): - dct = {k: self._merged[k] for k in self._merged} - pretty_vars = json.dumps(dct, sort_keys=True, indent=4) - msg = self.UndefinedVarError.format(var_name, self.node_name, pretty_vars) - raise_compiler_error(msg, self._node) + raise RequiredVarNotFound(var_name, self._merged, self._node) def has_var(self, var_name: str): return var_name in self._merged @@ -300,7 +297,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: """ return_value = None if var.startswith(SECRET_ENV_PREFIX): - disallow_secret_env_var(var) + raise DisallowSecretEnvVar(var) if var in os.environ: return_value = os.environ[var] elif default is not None: @@ -315,8 +312,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) if os.environ.get("DBT_MACRO_DEBUGGING"): @@ -497,7 +493,7 @@ def set_strict(value: Iterable[Any]) -> Set[Any]: try: return set(value) except TypeError as e: - raise CompilationException(e) + raise SetStrictWrongType(e) @contextmember("zip") @staticmethod @@ -541,7 +537,7 @@ def zip_strict(*args: Iterable[Any]) -> Iterable[Any]: try: return zip(*args) except TypeError as e: - raise CompilationException(e) + raise ZipStrictWrongType(e) @contextmember @staticmethod diff --git a/core/dbt/context/configured.py b/core/dbt/context/configured.py index ae2ee10baec..ca1de35423b 100644 --- a/core/dbt/context/configured.py +++ b/core/dbt/context/configured.py @@ -8,7 +8,7 @@ from dbt.context.base import contextproperty, contextmember, Var from dbt.context.target import TargetContext -from dbt.exceptions import raise_parsing_error, disallow_secret_env_var +from dbt.exceptions import EnvVarMissing, DisallowSecretEnvVar class ConfiguredContext(TargetContext): @@ -86,7 +86,7 @@ def var(self) -> ConfiguredVar: def env_var(self, var: str, default: Optional[str] = None) -> str: return_value = None if var.startswith(SECRET_ENV_PREFIX): - disallow_secret_env_var(var) + raise DisallowSecretEnvVar(var) if var in os.environ: return_value = os.environ[var] elif default is not None: @@ -104,8 +104,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) class MacroResolvingContext(ConfiguredContext): diff --git a/core/dbt/context/docs.py b/core/dbt/context/docs.py index 4908829d414..89a652736dd 100644 --- a/core/dbt/context/docs.py +++ b/core/dbt/context/docs.py @@ -1,8 +1,8 @@ from typing import Any, Dict, Union from dbt.exceptions import ( - doc_invalid_args, - doc_target_not_found, + DocTargetNotFound, + InvalidDocArgs, ) from dbt.config.runtime import RuntimeConfig from dbt.contracts.graph.manifest import Manifest @@ -52,7 +52,7 @@ def doc(self, *args: str) -> str: elif len(args) == 2: doc_package_name, doc_name = args else: - doc_invalid_args(self.node, args) + raise InvalidDocArgs(self.node, args) # Documentation target_doc = self.manifest.resolve_doc( @@ -68,7 +68,9 @@ def doc(self, *args: str) -> str: # TODO CT-211 source_file.add_node(self.node.unique_id) # type: ignore[union-attr] else: - doc_target_not_found(self.node, doc_name, doc_package_name) + raise DocTargetNotFound( + node=self.node, target_doc_name=doc_name, target_doc_package=doc_package_name + ) return target_doc.block_contents diff --git a/core/dbt/context/exceptions_jinja.py b/core/dbt/context/exceptions_jinja.py new file mode 100644 index 00000000000..5663b4701e0 --- /dev/null +++ b/core/dbt/context/exceptions_jinja.py @@ -0,0 +1,142 @@ +import functools +from typing import NoReturn + +from dbt.events.functions import warn_or_error +from dbt.events.helpers import env_secrets, scrub_secrets +from dbt.events.types import JinjaLogWarning + +from dbt.exceptions import ( + RuntimeException, + MissingConfig, + MissingMaterialization, + MissingRelation, + AmbiguousAlias, + AmbiguousCatalogMatch, + CacheInconsistency, + DataclassNotDict, + CompilationException, + DatabaseException, + DependencyNotFound, + DependencyException, + DuplicatePatchPath, + DuplicateResourceName, + InvalidPropertyYML, + NotImplementedException, + RelationWrongType, +) + + +def warn(msg, node=None): + warn_or_error(JinjaLogWarning(msg=msg), node=node) + return "" + + +def missing_config(model, name) -> NoReturn: + raise MissingConfig(unique_id=model.unique_id, name=name) + + +def missing_materialization(model, adapter_type) -> NoReturn: + raise MissingMaterialization(model=model, adapter_type=adapter_type) + + +def missing_relation(relation, model=None) -> NoReturn: + raise MissingRelation(relation, model) + + +def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn: + raise AmbiguousAlias(node_1, node_2, duped_name) + + +def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: + raise AmbiguousCatalogMatch(unique_id, match_1, match_2) + + +def raise_cache_inconsistent(message) -> NoReturn: + raise CacheInconsistency(message) + + +def raise_dataclass_not_dict(obj) -> NoReturn: + raise DataclassNotDict(obj) + + +def raise_compiler_error(msg, node=None) -> NoReturn: + raise CompilationException(msg, node) + + +def raise_database_error(msg, node=None) -> NoReturn: + raise DatabaseException(msg, node) + + +def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn: + raise DependencyNotFound(node, node_description, required_pkg) + + +def raise_dependency_error(msg) -> NoReturn: + raise DependencyException(scrub_secrets(msg, env_secrets())) + + +def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn: + raise DuplicatePatchPath(patch_1, existing_patch_path) + + +def raise_duplicate_resource_name(node_1, node_2) -> NoReturn: + raise DuplicateResourceName(node_1, node_2) + + +def raise_invalid_property_yml_version(path, issue) -> NoReturn: + raise InvalidPropertyYML(path, issue) + + +def raise_not_implemented(msg) -> NoReturn: + raise NotImplementedException(msg) + + +def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: + raise RelationWrongType(relation, expected_type, model) + + +# Update this when a new function should be added to the +# dbt context's `exceptions` key! +CONTEXT_EXPORTS = { + fn.__name__: fn + for fn in [ + warn, + missing_config, + missing_materialization, + missing_relation, + raise_ambiguous_alias, + raise_ambiguous_catalog_match, + raise_cache_inconsistent, + raise_dataclass_not_dict, + raise_compiler_error, + raise_database_error, + raise_dep_not_found, + raise_dependency_error, + raise_duplicate_patch_name, + raise_duplicate_resource_name, + raise_invalid_property_yml_version, + raise_not_implemented, + relation_wrong_type, + ] +} + + +# wraps context based exceptions in node info +def wrapper(model): + def wrap(func): + @functools.wraps(func) + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except RuntimeException as exc: + exc.add_node(model) + raise exc + + return inner + + return wrap + + +def wrapped_exports(model): + wrap = wrapper(model) + return {name: wrap(export) for name, export in CONTEXT_EXPORTS.items()} diff --git a/core/dbt/context/macro_resolver.py b/core/dbt/context/macro_resolver.py index a108a1889b9..6e70bafd05e 100644 --- a/core/dbt/context/macro_resolver.py +++ b/core/dbt/context/macro_resolver.py @@ -1,6 +1,6 @@ from typing import Dict, MutableMapping, Optional from dbt.contracts.graph.nodes import Macro -from dbt.exceptions import raise_duplicate_macro_name, raise_compiler_error +from dbt.exceptions import DuplicateMacroName, PackageNotFoundForMacro from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME from dbt.clients.jinja import MacroGenerator @@ -86,7 +86,7 @@ def _add_macro_to( package_namespaces[macro.package_name] = namespace if macro.name in namespace: - raise_duplicate_macro_name(macro, macro, macro.package_name) + raise DuplicateMacroName(macro, macro, macro.package_name) package_namespaces[macro.package_name][macro.name] = macro def add_macro(self, macro: Macro): @@ -187,7 +187,7 @@ def get_from_package(self, package_name: Optional[str], name: str) -> Optional[M elif package_name in self.macro_resolver.packages: macro = self.macro_resolver.packages[package_name].get(name) else: - raise_compiler_error(f"Could not find package '{package_name}'") + raise PackageNotFoundForMacro(package_name) if not macro: return None macro_func = MacroGenerator(macro, self.ctx, self.node, self.thread_ctx) diff --git a/core/dbt/context/macros.py b/core/dbt/context/macros.py index 700109b8081..921480ec05a 100644 --- a/core/dbt/context/macros.py +++ b/core/dbt/context/macros.py @@ -3,7 +3,7 @@ from dbt.clients.jinja import MacroGenerator, MacroStack from dbt.contracts.graph.nodes import Macro from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME -from dbt.exceptions import raise_duplicate_macro_name, raise_compiler_error +from dbt.exceptions import DuplicateMacroName, PackageNotFoundForMacro FlatNamespace = Dict[str, MacroGenerator] @@ -75,7 +75,7 @@ def get_from_package(self, package_name: Optional[str], name: str) -> Optional[M elif package_name in self.packages: return self.packages[package_name].get(name) else: - raise_compiler_error(f"Could not find package '{package_name}'") + raise PackageNotFoundForMacro(package_name) # This class builds the MacroNamespace by adding macros to @@ -122,7 +122,7 @@ def _add_macro_to( hierarchy[macro.package_name] = namespace if macro.name in namespace: - raise_duplicate_macro_name(macro_func.macro, macro, macro.package_name) + raise DuplicateMacroName(macro_func.macro, macro, macro.package_name) hierarchy[macro.package_name][macro.name] = macro_func def add_macro(self, macro: Macro, ctx: Dict[str, Any]): diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 06642810730..2e7af0a79f2 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -19,13 +19,14 @@ from dbt.clients import agate_helper from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack from dbt.config import RuntimeConfig, Project -from .base import contextmember, contextproperty, Var -from .configured import FQNLookup -from .context_config import ContextConfig from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER +from dbt.context.base import contextmember, contextproperty, Var +from dbt.context.configured import FQNLookup +from dbt.context.context_config import ContextConfig +from dbt.context.exceptions_jinja import wrapped_exports from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace -from .macros import MacroNamespaceBuilder, MacroNamespace -from .manifest import ManifestContext +from dbt.context.macros import MacroNamespaceBuilder, MacroNamespace +from dbt.context.manifest import ManifestContext from dbt.contracts.connection import AdapterResponse from dbt.contracts.graph.manifest import Manifest, Disabled from dbt.contracts.graph.nodes import ( @@ -41,20 +42,27 @@ from dbt.events.functions import get_metadata_vars from dbt.exceptions import ( CompilationException, - ParsingException, + ConflictingConfigKeys, + DisallowSecretEnvVar, + EnvVarMissing, InternalException, - ValidationException, + InvalidInlineModelConfig, + InvalidNumberSourceArgs, + InvalidPersistDocsValueType, + LoadAgateTableNotSeed, + LoadAgateTableValueError, + MacroInvalidDispatchArg, + MacrosSourcesUnWriteable, + MetricInvalidArgs, + MissingConfig, + OperationsCannotRefEphemeralNodes, + PackageNotInDeps, + ParsingException, + RefBadContext, + RefInvalidArgs, RuntimeException, - macro_invalid_dispatch_arg, - missing_config, - raise_compiler_error, - ref_invalid_args, - metric_invalid_args, - target_not_found, - ref_bad_context, - wrapped_exports, - raise_parsing_error, - disallow_secret_env_var, + TargetNotFound, + ValidationException, ) from dbt.config import IsFQNResource from dbt.node_types import NodeType, ModelLanguage @@ -139,7 +147,7 @@ def dispatch( raise CompilationException(msg) if packages is not None: - raise macro_invalid_dispatch_arg(macro_name) + raise MacroInvalidDispatchArg(macro_name) namespace = macro_namespace @@ -233,7 +241,7 @@ def __call__(self, *args: str) -> RelationProxy: elif len(args) == 2: package, name = args else: - ref_invalid_args(self.model, args) + raise RefInvalidArgs(node=self.model, args=args) self.validate_args(name, package) return self.resolve(name, package) @@ -257,9 +265,7 @@ def validate_args(self, source_name: str, table_name: str): def __call__(self, *args: str) -> RelationProxy: if len(args) != 2: - raise_compiler_error( - f"source() takes exactly two arguments ({len(args)} given)", self.model - ) + raise InvalidNumberSourceArgs(args, node=self.model) self.validate_args(args[0], args[1]) return self.resolve(args[0], args[1]) @@ -294,7 +300,7 @@ def __call__(self, *args: str) -> MetricReference: elif len(args) == 2: package, name = args else: - metric_invalid_args(self.model, args) + raise MetricInvalidArgs(node=self.model, args=args) self.validate_args(name, package) return self.resolve(name, package) @@ -315,12 +321,7 @@ def _transform_config(self, config): if oldkey in config: newkey = oldkey.replace("_", "-") if newkey in config: - raise_compiler_error( - 'Invalid config, has conflicting keys "{}" and "{}"'.format( - oldkey, newkey - ), - self.model, - ) + raise ConflictingConfigKeys(oldkey, newkey, node=self.model) config[newkey] = config.pop(oldkey) return config @@ -330,7 +331,7 @@ def __call__(self, *args, **kwargs): elif len(args) == 0 and len(kwargs) > 0: opts = kwargs else: - raise_compiler_error("Invalid inline model config", self.model) + raise InvalidInlineModelConfig(node=self.model) opts = self._transform_config(opts) @@ -378,7 +379,7 @@ def _lookup(self, name, default=_MISSING): else: result = self.model.config.get(name, default) if result is _MISSING: - missing_config(self.model, name) + raise MissingConfig(unique_id=self.model.unique_id, name=name) return result def require(self, name, validator=None): @@ -400,20 +401,14 @@ def get(self, name, default=None, validator=None): def persist_relation_docs(self) -> bool: persist_docs = self.get("persist_docs", default={}) if not isinstance(persist_docs, dict): - raise_compiler_error( - f"Invalid value provided for 'persist_docs'. Expected dict " - f"but received {type(persist_docs)}" - ) + raise InvalidPersistDocsValueType(persist_docs) return persist_docs.get("relation", False) def persist_column_docs(self) -> bool: persist_docs = self.get("persist_docs", default={}) if not isinstance(persist_docs, dict): - raise_compiler_error( - f"Invalid value provided for 'persist_docs'. Expected dict " - f"but received {type(persist_docs)}" - ) + raise InvalidPersistDocsValueType(persist_docs) return persist_docs.get("columns", False) @@ -472,7 +467,7 @@ def resolve(self, target_name: str, target_package: Optional[str] = None) -> Rel ) if target_model is None or isinstance(target_model, Disabled): - target_not_found( + raise TargetNotFound( node=self.model, target_name=target_name, target_kind="node", @@ -494,7 +489,7 @@ def validate( ) -> None: if resolved.unique_id not in self.model.depends_on.nodes: args = self._repack_args(target_name, target_package) - ref_bad_context(self.model, args) + raise RefBadContext(node=self.model, args=args) class OperationRefResolver(RuntimeRefResolver): @@ -510,12 +505,7 @@ def create_relation(self, target_model: ManifestNode, name: str) -> RelationProx if target_model.is_ephemeral_model: # In operations, we can't ref() ephemeral nodes, because # Macros do not support set_cte - raise_compiler_error( - "Operations can not ref() ephemeral nodes, but {} is ephemeral".format( - target_model.name - ), - self.model, - ) + raise OperationsCannotRefEphemeralNodes(target_model.name, node=self.model) else: return super().create_relation(target_model, name) @@ -538,7 +528,7 @@ def resolve(self, source_name: str, table_name: str): ) if target_source is None or isinstance(target_source, Disabled): - target_not_found( + raise TargetNotFound( node=self.model, target_name=f"{source_name}.{table_name}", target_kind="source", @@ -565,7 +555,7 @@ def resolve(self, target_name: str, target_package: Optional[str] = None) -> Met ) if target_metric is None or isinstance(target_metric, Disabled): - target_not_found( + raise TargetNotFound( node=self.model, target_name=target_name, target_kind="metric", @@ -594,7 +584,7 @@ def packages_for_node(self) -> Iterable[Project]: if package_name != self._config.project_name: if package_name not in dependencies: # I don't think this is actually reachable - raise_compiler_error(f"Node package named {package_name} not found!", self._node) + raise PackageNotInDeps(package_name, node=self._node) yield dependencies[package_name] yield self._config @@ -777,7 +767,7 @@ def inner(value: T) -> None: def write(self, payload: str) -> str: # macros/source defs aren't 'writeable'. if isinstance(self.model, (Macro, SourceDefinition)): - raise_compiler_error('cannot "write" macros or sources') + raise MacrosSourcesUnWriteable(node=self.model) self.model.build_path = self.model.write_node(self.config.target_path, "run", payload) return "" @@ -792,21 +782,19 @@ def try_or_compiler_error( try: return func(*args, **kwargs) except Exception: - raise_compiler_error(message_if_exception, self.model) + raise CompilationException(message_if_exception, self.model) @contextmember def load_agate_table(self) -> agate.Table: if not isinstance(self.model, SeedNode): - raise_compiler_error( - "can only load_agate_table for seeds (got a {})".format(self.model.resource_type) - ) + raise LoadAgateTableNotSeed(self.model.resource_type, node=self.model) assert self.model.root_path path = os.path.join(self.model.root_path, self.model.original_file_path) column_types = self.model.config.column_types try: table = agate_helper.from_csv(path, text_columns=column_types) except ValueError as e: - raise_compiler_error(str(e)) + raise LoadAgateTableValueError(e, node=self.model) table.original_abspath = os.path.abspath(path) return table @@ -1208,7 +1196,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: """ return_value = None if var.startswith(SECRET_ENV_PREFIX): - disallow_secret_env_var(var) + raise DisallowSecretEnvVar(var) if var in os.environ: return_value = os.environ[var] elif default is not None: @@ -1241,8 +1229,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: source_file.env_vars.append(var) # type: ignore[union-attr] return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) @contextproperty def selected_resources(self) -> List[str]: @@ -1423,7 +1410,7 @@ def generate_runtime_macro_context( class ExposureRefResolver(BaseResolver): def __call__(self, *args) -> str: if len(args) not in (1, 2): - ref_invalid_args(self.model, args) + raise RefInvalidArgs(node=self.model, args=args) self.model.refs.append(list(args)) return "" @@ -1431,9 +1418,7 @@ def __call__(self, *args) -> str: class ExposureSourceResolver(BaseResolver): def __call__(self, *args) -> str: if len(args) != 2: - raise_compiler_error( - f"source() takes exactly two arguments ({len(args)} given)", self.model - ) + raise InvalidNumberSourceArgs(args, node=self.model) self.model.sources.append(list(args)) return "" @@ -1441,7 +1426,7 @@ def __call__(self, *args) -> str: class ExposureMetricResolver(BaseResolver): def __call__(self, *args) -> str: if len(args) not in (1, 2): - metric_invalid_args(self.model, args) + raise MetricInvalidArgs(node=self.model, args=args) self.model.metrics.append(list(args)) return "" @@ -1483,7 +1468,7 @@ def __call__(self, *args) -> str: elif len(args) == 2: package, name = args else: - ref_invalid_args(self.model, args) + raise RefInvalidArgs(node=self.model, args=args) self.validate_args(name, package) self.model.refs.append(list(args)) return "" @@ -1573,7 +1558,7 @@ def _build_test_namespace(self): def env_var(self, var: str, default: Optional[str] = None) -> str: return_value = None if var.startswith(SECRET_ENV_PREFIX): - disallow_secret_env_var(var) + raise DisallowSecretEnvVar(var) if var in os.environ: return_value = os.environ[var] elif default is not None: @@ -1599,8 +1584,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: source_file.add_env_var(var, yaml_key, name) # type: ignore[union-attr] return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) def generate_test_context( diff --git a/core/dbt/context/secret.py b/core/dbt/context/secret.py index 11a6dc54f07..da13509ef50 100644 --- a/core/dbt/context/secret.py +++ b/core/dbt/context/secret.py @@ -4,7 +4,7 @@ from .base import BaseContext, contextmember from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER -from dbt.exceptions import raise_parsing_error +from dbt.exceptions import EnvVarMissing SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$" @@ -50,8 +50,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: self.env_vars[var] = return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) def generate_secret_context(cli_vars: Dict[str, Any]) -> Dict[str, Any]: diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index cd1eb561fcc..c43012ec521 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -41,14 +41,14 @@ from dbt.dataclass_schema import dbtClassMixin from dbt.exceptions import ( CompilationException, - raise_duplicate_resource_name, - raise_compiler_error, + DuplicateResourceName, + DuplicateMacroInPackage, + DuplicateMaterializationName, ) from dbt.helper_types import PathSet from dbt.events.functions import fire_event from dbt.events.types import MergedFromState from dbt.node_types import NodeType -from dbt.ui import line_wrap_message from dbt import flags from dbt import tracking import dbt.utils @@ -398,12 +398,7 @@ def __eq__(self, other: object) -> bool: return NotImplemented equal = self.specificity == other.specificity and self.locality == other.locality if equal: - raise_compiler_error( - "Found two materializations with the name {} (packages {} and " - "{}). dbt cannot resolve this ambiguity".format( - self.macro.name, self.macro.package_name, other.macro.package_name - ) - ) + raise DuplicateMaterializationName(self.macro, other) return equal @@ -1040,26 +1035,7 @@ def merge_from_artifact( def add_macro(self, source_file: SourceFile, macro: Macro): if macro.unique_id in self.macros: # detect that the macro exists and emit an error - other_path = self.macros[macro.unique_id].original_file_path - # subtract 2 for the "Compilation Error" indent - # note that the line wrap eats newlines, so if you want newlines, - # this is the result :( - msg = line_wrap_message( - f"""\ - dbt found two macros named "{macro.name}" in the project - "{macro.package_name}". - - - To fix this error, rename or remove one of the following - macros: - - - {macro.original_file_path} - - - {other_path} - """, - subtract=2, - ) - raise_compiler_error(msg) + raise DuplicateMacroInPackage(macro=macro, macro_mapping=self.macros) self.macros[macro.unique_id] = macro source_file.macros.append(macro.unique_id) @@ -1237,7 +1213,7 @@ def __post_serialize__(self, dct): def _check_duplicates(value: BaseNode, src: Mapping[str, BaseNode]): if value.unique_id in src: - raise_duplicate_resource_name(value, src[value.unique_id]) + raise DuplicateResourceName(value, src[value.unique_id]) K_T = TypeVar("K_T") diff --git a/core/dbt/contracts/relation.py b/core/dbt/contracts/relation.py index fbe18146bb4..e8cba2ad155 100644 --- a/core/dbt/contracts/relation.py +++ b/core/dbt/contracts/relation.py @@ -9,7 +9,7 @@ from dbt.dataclass_schema import dbtClassMixin, StrEnum from dbt.contracts.util import Replaceable -from dbt.exceptions import raise_dataclass_not_dict, CompilationException +from dbt.exceptions import CompilationException, DataclassNotDict from dbt.utils import deep_merge @@ -43,10 +43,10 @@ def __getitem__(self, key): raise KeyError(key) from None def __iter__(self): - raise_dataclass_not_dict(self) + raise DataclassNotDict(self) def __len__(self): - raise_dataclass_not_dict(self) + raise DataclassNotDict(self) def incorporate(self, **kwargs): value = self.to_dict(omit_none=True) diff --git a/core/dbt/deps/git.py b/core/dbt/deps/git.py index e6dcc479a80..5d7a1331c58 100644 --- a/core/dbt/deps/git.py +++ b/core/dbt/deps/git.py @@ -9,7 +9,7 @@ GitPackage, ) from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path -from dbt.exceptions import ExecutableError, raise_dependency_error +from dbt.exceptions import ExecutableError, MultipleVersionGitDeps from dbt.events.functions import fire_event, warn_or_error from dbt.events.types import EnsureGitInstalled, DepsUnpinned @@ -143,10 +143,7 @@ def resolved(self) -> GitPinnedPackage: if len(requested) == 0: requested = {"HEAD"} elif len(requested) > 1: - raise_dependency_error( - "git dependencies should contain exactly one version. " - "{} contains: {}".format(self.git, requested) - ) + raise MultipleVersionGitDeps(self.git, requested) return GitPinnedPackage( git=self.git, diff --git a/core/dbt/deps/registry.py b/core/dbt/deps/registry.py index 9f163d89758..f3398f4b16f 100644 --- a/core/dbt/deps/registry.py +++ b/core/dbt/deps/registry.py @@ -10,10 +10,10 @@ ) from dbt.deps.base import PinnedPackage, UnpinnedPackage from dbt.exceptions import ( - package_version_not_found, - VersionsNotCompatibleException, DependencyException, - package_not_found, + PackageNotFound, + PackageVersionNotFound, + VersionsNotCompatibleException, ) @@ -71,7 +71,7 @@ def __init__( def _check_in_index(self): index = registry.index_cached() if self.package not in index: - package_not_found(self.package) + raise PackageNotFound(self.package) @classmethod def from_contract(cls, contract: RegistryPackage) -> "RegistryUnpinnedPackage": @@ -118,7 +118,7 @@ def resolved(self) -> RegistryPinnedPackage: target = None if not target: # raise an exception if no installable target version is found - package_version_not_found(self.package, range_, installable, should_version_check) + raise PackageVersionNotFound(self.package, range_, installable, should_version_check) latest_compatible = installable[-1] return RegistryPinnedPackage( package=self.package, version=target, version_latest=latest_compatible diff --git a/core/dbt/deps/resolver.py b/core/dbt/deps/resolver.py index e4c1992894c..323e2f562c1 100644 --- a/core/dbt/deps/resolver.py +++ b/core/dbt/deps/resolver.py @@ -1,7 +1,12 @@ from dataclasses import dataclass, field from typing import Dict, List, NoReturn, Union, Type, Iterator, Set -from dbt.exceptions import raise_dependency_error, InternalException +from dbt.exceptions import ( + DuplicateDependencyToRoot, + DuplicateProjectDependency, + MismatchedDependencyTypes, + InternalException, +) from dbt.config import Project, RuntimeConfig from dbt.config.renderer import DbtProjectYamlRenderer @@ -51,10 +56,7 @@ def __setitem__(self, key: BasePackage, value): self.packages[key_str] = value def _mismatched_types(self, old: UnpinnedPackage, new: UnpinnedPackage) -> NoReturn: - raise_dependency_error( - f"Cannot incorporate {new} ({new.__class__.__name__}) in {old} " - f"({old.__class__.__name__}): mismatched types" - ) + raise MismatchedDependencyTypes(new, old) def incorporate(self, package: UnpinnedPackage): key: str = self._pick_key(package) @@ -105,17 +107,9 @@ def _check_for_duplicate_project_names( for package in final_deps: project_name = package.get_project_name(config, renderer) if project_name in seen: - raise_dependency_error( - f'Found duplicate project "{project_name}". This occurs when ' - "a dependency has the same project name as some other " - "dependency." - ) + raise DuplicateProjectDependency(project_name) elif project_name == config.project_name: - raise_dependency_error( - "Found a dependency with the same name as the root project " - f'"{project_name}". Package names must be unique in a project.' - " Please rename one of these packages." - ) + raise DuplicateDependencyToRoot(project_name) seen.add(project_name) diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 36dd2e9ba79..f061606632e 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -159,9 +159,10 @@ def event_to_dict(event: BaseEvent) -> dict: def warn_or_error(event, node=None): if flags.WARN_ERROR: - from dbt.exceptions import raise_compiler_error + # TODO: resolve this circular import when at top + from dbt.exceptions import EventCompilationException - raise_compiler_error(scrub_secrets(event.info.msg, env_secrets()), node) + raise EventCompilationException(event.info.msg, node) else: fire_event(event) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 32aa8b477a9..2db130bb44e 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1,23 +1,29 @@ import builtins -import functools -from typing import NoReturn, Optional, Mapping, Any +import json +import re +from typing import Any, Dict, List, Mapping, NoReturn, Optional, Union +# from dbt.contracts.graph import ManifestNode # or ParsedNode? +from dbt.dataclass_schema import ValidationError +from dbt.events.functions import warn_or_error from dbt.events.helpers import env_secrets, scrub_secrets from dbt.events.types import JinjaLogWarning from dbt.events.contextvars import get_node_info from dbt.node_types import NodeType +from dbt.ui import line_wrap_message import dbt.dataclass_schema -def validator_error_message(exc): - """Given a dbt.dataclass_schema.ValidationError (which is basically a - jsonschema.ValidationError), return the relevant parts as a string +class MacroReturn(builtins.BaseException): + """ + Hack of all hacks + This is not actually an exception. + It's how we return a value from a macro. """ - if not isinstance(exc, dbt.dataclass_schema.ValidationError): - return str(exc) - path = "[%s]" % "][".join(map(repr, exc.relative_path)) - return "at path {}: {}".format(path, exc.message) + + def __init__(self, value): + self.value = value class Exception(builtins.Exception): @@ -32,25 +38,53 @@ def data(self): } -class MacroReturn(builtins.BaseException): - """ - Hack of all hacks - """ +class InternalException(Exception): + def __init__(self, msg: str): + self.stack: List = [] + self.msg = scrub_secrets(msg, env_secrets()) - def __init__(self, value): - self.value = value + @property + def type(self): + return "Internal" + def process_stack(self): + lines = [] + stack = self.stack + first = True -class InternalException(Exception): - pass + if len(stack) > 1: + lines.append("") + + for item in stack: + msg = "called by" + + if first: + msg = "in" + first = False + + lines.append(f"> {msg}") + + return lines + + def __str__(self): + if hasattr(self.msg, "split"): + split_msg = self.msg.split("\n") + else: + split_msg = str(self.msg).split("\n") + + lines = ["{}".format(self.type + " Error")] + split_msg + + lines += self.process_stack() + + return lines[0] + "\n" + "\n".join([" " + line for line in lines[1:]]) class RuntimeException(RuntimeError, Exception): CODE = 10001 MESSAGE = "Runtime error" - def __init__(self, msg, node=None): - self.stack = [] + def __init__(self, msg: str, node=None): + self.stack: List = [] self.node = node self.msg = scrub_secrets(msg, env_secrets()) @@ -69,14 +103,14 @@ def node_to_string(self, node): return "" if not hasattr(node, "name"): # we probably failed to parse a block, so we can't know the name - return "{} ({})".format(node.resource_type, node.original_file_path) + return f"{node.resource_type} ({node.original_file_path})" if hasattr(node, "contents"): # handle FileBlocks. They aren't really nodes but we want to render # out the path we know at least. This indicates an error during # block parsing. - return "{}".format(node.path.original_file_path) - return "{} {} ({})".format(node.resource_type, node.name, node.original_file_path) + return f"{node.path.original_file_path}" + return f"{node.resource_type} {node.name} ({node.original_file_path})" def process_stack(self): lines = [] @@ -93,15 +127,24 @@ def process_stack(self): msg = "in" first = False - lines.append("> {} {}".format(msg, self.node_to_string(item))) + lines.append(f"> {msg} {self.node_to_string(item)}") return lines - def __str__(self, prefix="! "): + def validator_error_message(self, exc: builtins.Exception): + """Given a dbt.dataclass_schema.ValidationError (which is basically a + jsonschema.ValidationError), return the relevant parts as a string + """ + if not isinstance(exc, dbt.dataclass_schema.ValidationError): + return str(exc) + path = "[%s]" % "][".join(map(repr, exc.relative_path)) + return f"at path {path}: {exc.message}" + + def __str__(self, prefix: str = "! "): node_string = "" if self.node is not None: - node_string = " in {}".format(self.node_to_string(self.node)) + node_string = f" in {self.node_to_string(self.node)}" if hasattr(self.msg, "split"): split_msg = self.msg.split("\n") @@ -138,7 +181,7 @@ class RPCTimeoutException(RuntimeException): CODE = 10008 MESSAGE = "RPC timeout error" - def __init__(self, timeout): + def __init__(self, timeout: Optional[float]): super().__init__(self.MESSAGE) self.timeout = timeout @@ -147,7 +190,7 @@ def data(self): result.update( { "timeout": self.timeout, - "message": "RPC timed out after {}s".format(self.timeout), + "message": f"RPC timed out after {self.timeout}s", } ) return result @@ -157,15 +200,15 @@ class RPCKilledException(RuntimeException): CODE = 10009 MESSAGE = "RPC process killed" - def __init__(self, signum): + def __init__(self, signum: int): self.signum = signum - self.message = "RPC process killed by signal {}".format(self.signum) - super().__init__(self.message) + self.msg = f"RPC process killed by signal {self.signum}" + super().__init__(self.msg) def data(self): return { "signum": self.signum, - "message": self.message, + "message": self.msg, } @@ -173,7 +216,7 @@ class RPCCompiling(RuntimeException): CODE = 10010 MESSAGE = 'RPC server is compiling the project, call the "status" method for' " compile status" - def __init__(self, msg=None, node=None): + def __init__(self, msg: str = None, node=None): if msg is None: msg = "compile in progress" super().__init__(msg, node) @@ -185,13 +228,13 @@ class RPCLoadException(RuntimeException): 'RPC server failed to compile project, call the "status" method for' " compile status" ) - def __init__(self, cause): + def __init__(self, cause: Dict[str, Any]): self.cause = cause - self.message = "{}: {}".format(self.MESSAGE, self.cause["message"]) - super().__init__(self.message) + self.msg = f'{self.MESSAGE}: {self.cause["message"]}' + super().__init__(self.msg) def data(self): - return {"cause": self.cause, "message": self.message} + return {"cause": self.cause, "message": self.msg} class DatabaseException(RuntimeException): @@ -202,7 +245,7 @@ def process_stack(self): lines = [] if hasattr(self.node, "build_path") and self.node.build_path: - lines.append("compiled Code at {}".format(self.node.build_path)) + lines.append(f"compiled Code at {self.node.build_path}") return lines + RuntimeException.process_stack(self) @@ -219,6 +262,17 @@ class CompilationException(RuntimeException): def type(self): return "Compilation" + def _fix_dupe_msg(self, path_1: str, path_2: str, name: str, type_name: str) -> str: + if path_1 == path_2: + return ( + f"remove one of the {type_name} entries for {name} in this file:\n - {path_1!s}\n" + ) + else: + return ( + f"remove the {type_name} entry for {name} in one of these files:\n" + f" - {path_1!s}\n{path_2!s}" + ) + class RecursionException(RuntimeException): pass @@ -238,14 +292,13 @@ def type(self): return "Parsing" +# TODO: this isn't raised in the core codebase. Is it raised elsewhere? class JSONValidationException(ValidationException): def __init__(self, typename, errors): self.typename = typename self.errors = errors self.errors_message = ", ".join(errors) - msg = 'Invalid arguments passed to "{}" instance: {}'.format( - self.typename, self.errors_message - ) + msg = f'Invalid arguments passed to "{self.typename}" instance: {self.errors_message}' super().__init__(msg) def __reduce__(self): @@ -259,7 +312,7 @@ def __init__(self, expected: str, found: Optional[str]): self.found = found self.filename = "input file" - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def add_filename(self, filename: str): self.filename = filename @@ -286,7 +339,7 @@ class JinjaRenderingException(CompilationException): class UndefinedMacroException(CompilationException): - def __str__(self, prefix="! ") -> str: + def __str__(self, prefix: str = "! ") -> str: msg = super().__str__(prefix) return ( f"{msg}. This can happen when calling a macro that does " @@ -303,7 +356,7 @@ def __init__(self, task_id): self.task_id = task_id def __str__(self): - return "{}: {}".format(self.MESSAGE, self.task_id) + return f"{self.MESSAGE}: {self.task_id}" class AliasException(ValidationException): @@ -320,9 +373,9 @@ class DbtConfigError(RuntimeException): CODE = 10007 MESSAGE = "DBT Configuration Error" - def __init__(self, message, project=None, result_type="invalid_project", path=None): + def __init__(self, msg: str, project=None, result_type="invalid_project", path=None): self.project = project - super().__init__(message) + super().__init__(msg) self.result_type = result_type self.path = path @@ -338,8 +391,8 @@ class FailFastException(RuntimeException): CODE = 10013 MESSAGE = "FailFast Error" - def __init__(self, message, result=None, node=None): - super().__init__(msg=message, node=node) + def __init__(self, msg: str, result=None, node=None): + super().__init__(msg=msg, node=node) self.result = result @property @@ -360,7 +413,7 @@ class DbtProfileError(DbtConfigError): class SemverException(Exception): - def __init__(self, msg=None): + def __init__(self, msg: str = None): self.msg = msg if msg is not None: super().__init__(msg) @@ -373,7 +426,10 @@ class VersionsNotCompatibleException(SemverException): class NotImplementedException(Exception): - pass + def __init__(self, msg: str): + self.msg = msg + self.formatted_msg = f"ERROR: {self.msg}" + super().__init__(self.formatted_msg) class FailedToConnectException(DatabaseException): @@ -381,52 +437,58 @@ class FailedToConnectException(DatabaseException): class CommandError(RuntimeException): - def __init__(self, cwd, cmd, message="Error running command"): + def __init__(self, cwd: str, cmd: List[str], msg: str = "Error running command"): cmd_scrubbed = list(scrub_secrets(cmd_txt, env_secrets()) for cmd_txt in cmd) - super().__init__(message) + super().__init__(msg) self.cwd = cwd self.cmd = cmd_scrubbed - self.args = (cwd, cmd_scrubbed, message) + self.args = (cwd, cmd_scrubbed, msg) def __str__(self): if len(self.cmd) == 0: - return "{}: No arguments given".format(self.msg) - return '{}: "{}"'.format(self.msg, self.cmd[0]) + return f"{self.msg}: No arguments given" + return f'{self.msg}: "{self.cmd[0]}"' class ExecutableError(CommandError): - def __init__(self, cwd, cmd, message): - super().__init__(cwd, cmd, message) + def __init__(self, cwd: str, cmd: List[str], msg: str): + super().__init__(cwd, cmd, msg) class WorkingDirectoryError(CommandError): - def __init__(self, cwd, cmd, message): - super().__init__(cwd, cmd, message) + def __init__(self, cwd: str, cmd: List[str], msg: str): + super().__init__(cwd, cmd, msg) def __str__(self): - return '{}: "{}"'.format(self.msg, self.cwd) + return f'{self.msg}: "{self.cwd}"' class CommandResultError(CommandError): - def __init__(self, cwd, cmd, returncode, stdout, stderr, message="Got a non-zero returncode"): - super().__init__(cwd, cmd, message) + def __init__( + self, + cwd: str, + cmd: List[str], + returncode: Union[int, Any], + stdout: bytes, + stderr: bytes, + msg: str = "Got a non-zero returncode", + ): + super().__init__(cwd, cmd, msg) self.returncode = returncode self.stdout = scrub_secrets(stdout.decode("utf-8"), env_secrets()) self.stderr = scrub_secrets(stderr.decode("utf-8"), env_secrets()) - self.args = (cwd, self.cmd, returncode, self.stdout, self.stderr, message) + self.args = (cwd, self.cmd, returncode, self.stdout, self.stderr, msg) def __str__(self): - return "{} running: {}".format(self.msg, self.cmd) + return f"{self.msg} running: {self.cmd}" class InvalidConnectionException(RuntimeException): - def __init__(self, thread_id, known, node=None): + def __init__(self, thread_id, known: List): self.thread_id = thread_id self.known = known super().__init__( - msg="connection never acquired for thread {}, have {}".format( - self.thread_id, self.known - ) + msg="connection never acquired for thread {self.thread_id}, have {self.known}" ) @@ -440,611 +502,1863 @@ class DuplicateYamlKeyException(CompilationException): pass -def raise_compiler_error(msg, node=None) -> NoReturn: - raise CompilationException(msg, node) +class ConnectionException(Exception): + """ + There was a problem with the connection that returned a bad response, + timed out, or resulted in a file that is corrupt. + """ + pass -def raise_parsing_error(msg, node=None) -> NoReturn: - raise ParsingException(msg, node) +# event level exception +class EventCompilationException(CompilationException): + def __init__(self, msg: str, node): + self.msg = scrub_secrets(msg, env_secrets()) + self.node = node + super().__init__(msg=self.msg) -def raise_database_error(msg, node=None) -> NoReturn: - raise DatabaseException(msg, node) +# compilation level exceptions +class GraphDependencyNotFound(CompilationException): + def __init__(self, node, dependency: str): + self.node = node + self.dependency = dependency + super().__init__(msg=self.get_message()) -def raise_dependency_error(msg) -> NoReturn: - raise DependencyException(scrub_secrets(msg, env_secrets())) + def get_message(self) -> str: + msg = f"'{self.node.unique_id}' depends on '{self.dependency}' which is not in the graph!" + return msg -def raise_git_cloning_error(error: CommandResultError) -> NoReturn: - error.cmd = scrub_secrets(str(error.cmd), env_secrets()) - raise error +# client level exceptions -def raise_git_cloning_problem(repo) -> NoReturn: - repo = scrub_secrets(repo, env_secrets()) - msg = """\ - Something went wrong while cloning {} - Check the debug logs for more information - """ - raise RuntimeException(msg.format(repo)) +class NoSupportedLanguagesFound(CompilationException): + def __init__(self, node): + self.node = node + self.msg = f"No supported_languages found in materialization macro {self.node.name}" + super().__init__(msg=self.msg) -def disallow_secret_env_var(env_var_name) -> NoReturn: - """Raise an error when a secret env var is referenced outside allowed - rendering contexts""" - msg = ( - "Secret env vars are allowed only in profiles.yml or packages.yml. " - "Found '{env_var_name}' referenced elsewhere." - ) - raise_parsing_error(msg.format(env_var_name=env_var_name)) +class MaterializtionMacroNotUsed(CompilationException): + def __init__(self, node): + self.node = node + self.msg = "Only materialization macros can be used with this function" + super().__init__(msg=self.msg) -def invalid_type_error( - method_name, arg_name, got_value, expected_type, version="0.13.0" -) -> NoReturn: - """Raise a CompilationException when an adapter method available to macros - has changed. - """ - got_type = type(got_value) - msg = ( - "As of {version}, 'adapter.{method_name}' expects argument " - "'{arg_name}' to be of type '{expected_type}', instead got " - "{got_value} ({got_type})" - ) - raise_compiler_error( - msg.format( - version=version, - method_name=method_name, - arg_name=arg_name, - expected_type=expected_type, - got_value=got_value, - got_type=got_type, - ) - ) +class UndefinedCompilation(CompilationException): + def __init__(self, name: str, node): + self.name = name + self.node = node + self.msg = f"{self.name} is undefined" + super().__init__(msg=self.msg) -def invalid_bool_error(got_value, macro_name) -> NoReturn: - """Raise a CompilationException when a macro expects a boolean but gets some - other value. - """ - msg = ( - "Macro '{macro_name}' returns '{got_value}'. It is not type 'bool' " - "and cannot not be converted reliably to a bool." - ) - raise_compiler_error(msg.format(macro_name=macro_name, got_value=got_value)) +class CaughtMacroExceptionWithNode(CompilationException): + def __init__(self, exc, node): + self.exc = exc + self.node = node + super().__init__(msg=str(exc)) -def ref_invalid_args(model, args) -> NoReturn: - raise_compiler_error("ref() takes at most two arguments ({} given)".format(len(args)), model) +class CaughtMacroException(CompilationException): + def __init__(self, exc): + self.exc = exc + super().__init__(msg=str(exc)) -def metric_invalid_args(model, args) -> NoReturn: - raise_compiler_error( - "metric() takes at most two arguments ({} given)".format(len(args)), model - ) +class MacroNameNotString(CompilationException): + def __init__(self, kwarg_value): + self.kwarg_value = kwarg_value + super().__init__(msg=self.get_message()) + def get_message(self) -> str: + msg = ( + f"The macro_name parameter ({self.kwarg_value}) " + "to adapter.dispatch was not a string" + ) + return msg -def ref_bad_context(model, args) -> NoReturn: - ref_args = ", ".join("'{}'".format(a) for a in args) - ref_string = "{{{{ ref({}) }}}}".format(ref_args) - base_error_msg = """dbt was unable to infer all dependencies for the model "{model_name}". -This typically happens when ref() is placed within a conditional block. +class MissingControlFlowStartTag(CompilationException): + def __init__(self, tag, expected_tag: str, tag_parser): + self.tag = tag + self.expected_tag = expected_tag + self.tag_parser = tag_parser + super().__init__(msg=self.get_message()) -To fix this, add the following hint to the top of the model "{model_name}": + def get_message(self) -> str: + linepos = self.tag_parser.linepos(self.tag.start) + msg = ( + f"Got an unexpected control flow end tag, got {self.tag.block_type_name} but " + f"expected {self.expected_tag} next (@ {linepos})" + ) + return msg --- depends_on: {ref_string}""" - # This explicitly references model['name'], instead of model['alias'], for - # better error messages. Ex. If models foo_users and bar_users are aliased - # to 'users', in their respective schemas, then you would want to see - # 'bar_users' in your error messge instead of just 'users'. - if isinstance(model, dict): # TODO: remove this path - model_name = model["name"] - model_path = model["path"] - else: - model_name = model.name - model_path = model.path - error_msg = base_error_msg.format( - model_name=model_name, model_path=model_path, ref_string=ref_string - ) - raise_compiler_error(error_msg, model) +class UnexpectedControlFlowEndTag(CompilationException): + def __init__(self, tag, expected_tag: str, tag_parser): + self.tag = tag + self.expected_tag = expected_tag + self.tag_parser = tag_parser + super().__init__(msg=self.get_message()) -def doc_invalid_args(model, args) -> NoReturn: - raise_compiler_error("doc() takes at most two arguments ({} given)".format(len(args)), model) + def get_message(self) -> str: + linepos = self.tag_parser.linepos(self.tag.start) + msg = ( + f"Got an unexpected control flow end tag, got {self.tag.block_type_name} but " + f"never saw a preceeding {self.expected_tag} (@ {linepos})" + ) + return msg -def doc_target_not_found( - model, target_doc_name: str, target_doc_package: Optional[str] -) -> NoReturn: - target_package_string = "" +class UnexpectedMacroEOF(CompilationException): + def __init__(self, expected_name: str, actual_name: str): + self.expected_name = expected_name + self.actual_name = actual_name + super().__init__(msg=self.get_message()) - if target_doc_package is not None: - target_package_string = "in package '{}' ".format(target_doc_package) + def get_message(self) -> str: + msg = f'unexpected EOF, expected {self.expected_name}, got "{self.actual_name}"' + return msg - msg = ("Documentation for '{}' depends on doc '{}' {} which was not found").format( - model.unique_id, target_doc_name, target_package_string - ) - raise_compiler_error(msg, model) +class MacroNamespaceNotString(CompilationException): + def __init__(self, kwarg_type: Any): + self.kwarg_type = kwarg_type + super().__init__(msg=self.get_message()) -def get_not_found_or_disabled_msg( - original_file_path, - unique_id, - resource_type_title, - target_name: str, - target_kind: str, - target_package: Optional[str] = None, - disabled: Optional[bool] = None, -) -> str: - if disabled is None: - reason = "was not found or is disabled" - elif disabled is True: - reason = "is disabled" - else: - reason = "was not found" - - target_package_string = "" - if target_package is not None: - target_package_string = "in package '{}' ".format(target_package) - - return "{} '{}' ({}) depends on a {} named '{}' {}which {}".format( - resource_type_title, - unique_id, - original_file_path, - target_kind, - target_name, - target_package_string, - reason, - ) + def get_message(self) -> str: + msg = ( + "The macro_namespace parameter to adapter.dispatch " + f"is a {self.kwarg_type}, not a string" + ) + return msg -def target_not_found( - node, - target_name: str, - target_kind: str, - target_package: Optional[str] = None, - disabled: Optional[bool] = None, -) -> NoReturn: - msg = get_not_found_or_disabled_msg( - original_file_path=node.original_file_path, - unique_id=node.unique_id, - resource_type_title=node.resource_type.title(), - target_name=target_name, - target_kind=target_kind, - target_package=target_package, - disabled=disabled, - ) +class NestedTags(CompilationException): + def __init__(self, outer, inner): + self.outer = outer + self.inner = inner + super().__init__(msg=self.get_message()) - raise_compiler_error(msg, node) + def get_message(self) -> str: + msg = ( + f"Got nested tags: {self.outer.block_type_name} (started at {self.outer.start}) did " + f"not have a matching {{{{% end{self.outer.block_type_name} %}}}} before a " + f"subsequent {self.inner.block_type_name} was found (started at {self.inner.start})" + ) + return msg -def dependency_not_found(model, target_model_name): - raise_compiler_error( - "'{}' depends on '{}' which is not in the graph!".format( - model.unique_id, target_model_name - ), - model, - ) +class BlockDefinitionNotAtTop(CompilationException): + def __init__(self, tag_parser, tag_start): + self.tag_parser = tag_parser + self.tag_start = tag_start + super().__init__(msg=self.get_message()) + def get_message(self) -> str: + position = self.tag_parser.linepos(self.tag_start) + msg = ( + f"Got a block definition inside control flow at {position}. " + "All dbt block definitions must be at the top level" + ) + return msg -def macro_not_found(model, target_macro_id): - raise_compiler_error( - model, - "'{}' references macro '{}' which is not defined!".format( - model.unique_id, target_macro_id - ), - ) +class MissingCloseTag(CompilationException): + def __init__(self, block_type_name: str, linecount: int): + self.block_type_name = block_type_name + self.linecount = linecount + super().__init__(msg=self.get_message()) -def macro_invalid_dispatch_arg(macro_name) -> NoReturn: - msg = """\ - The "packages" argument of adapter.dispatch() has been deprecated. - Use the "macro_namespace" argument instead. + def get_message(self) -> str: + msg = f"Reached EOF without finding a close tag for {self.block_type_name} (searched from line {self.linecount})" + return msg - Raised during dispatch for: {} - For more information, see: +class GitCloningProblem(RuntimeException): + def __init__(self, repo: str): + self.repo = scrub_secrets(repo, env_secrets()) + super().__init__(msg=self.get_message()) - https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch - """ - raise_compiler_error(msg.format(macro_name)) + def get_message(self) -> str: + msg = f"""\ + Something went wrong while cloning {self.repo} + Check the debug logs for more information + """ + return msg -def materialization_not_available(model, adapter_type): - materialization = model.get_materialization() +class GitCloningError(InternalException): + def __init__(self, repo: str, revision: str, error: CommandResultError): + self.repo = repo + self.revision = revision + self.error = error + super().__init__(msg=self.get_message()) - raise_compiler_error( - "Materialization '{}' is not available for {}!".format(materialization, adapter_type), - model, - ) + def get_message(self) -> str: + stderr = self.error.stderr.strip() + if "usage: git" in stderr: + stderr = stderr.split("\nusage: git")[0] + if re.match("fatal: destination path '(.+)' already exists", stderr): + self.error.cmd = list(scrub_secrets(str(self.error.cmd), env_secrets())) + raise self.error + msg = f"Error checking out spec='{self.revision}' for repo {self.repo}\n{stderr}" + return scrub_secrets(msg, env_secrets()) -def missing_materialization(model, adapter_type): - materialization = model.get_materialization() - valid_types = "'default'" +class GitCheckoutError(InternalException): + def __init__(self, repo: str, revision: str, error: CommandResultError): + self.repo = repo + self.revision = revision + self.stderr = error.stderr.strip() + super().__init__(msg=self.get_message()) - if adapter_type != "default": - valid_types = "'default' and '{}'".format(adapter_type) + def get_message(self) -> str: + msg = f"Error checking out spec='{self.revision}' for repo {self.repo}\n{self.stderr}" + return scrub_secrets(msg, env_secrets()) - raise_compiler_error( - "No materialization '{}' was found for adapter {}! (searched types {})".format( - materialization, adapter_type, valid_types - ), - model, - ) +class InvalidMaterializationArg(CompilationException): + def __init__(self, name: str, argument: str): + self.name = name + self.argument = argument + super().__init__(msg=self.get_message()) -def bad_package_spec(repo, spec, error_message): - msg = "Error checking out spec='{}' for repo {}\n{}".format(spec, repo, error_message) - raise InternalException(scrub_secrets(msg, env_secrets())) + def get_message(self) -> str: + msg = f"materialization '{self.name}' received unknown argument '{self.argument}'." + return msg -def raise_cache_inconsistent(message): - raise InternalException("Cache inconsistency detected: {}".format(message)) +class SymbolicLinkError(CompilationException): + def __init__(self): + super().__init__(msg=self.get_message()) + def get_message(self) -> str: + msg = ( + "dbt encountered an error when attempting to create a symbolic link. " + "If this error persists, please create an issue at: \n\n" + "https://github.com/dbt-labs/dbt-core" + ) -def missing_config(model, name): - raise_compiler_error( - "Model '{}' does not define a required config parameter '{}'.".format( - model.unique_id, name - ), - model, - ) + return msg -def missing_relation(relation, model=None): - raise_compiler_error("Relation {} not found!".format(relation), model) +# context level exceptions -def raise_dataclass_not_dict(obj): - msg = ( - 'The object ("{obj}") was used as a dictionary. This ' - "capability has been removed from objects of this type." - ) - raise_compiler_error(msg) +class ZipStrictWrongType(CompilationException): + def __init__(self, exc): + self.exc = exc + msg = str(self.exc) + super().__init__(msg=msg) -def relation_wrong_type(relation, expected_type, model=None): - raise_compiler_error( - ( - "Trying to create {expected_type} {relation}, " - "but it currently exists as a {current_type}. Either " - "drop {relation} manually, or run dbt with " - "`--full-refresh` and dbt will drop it for you." - ).format(relation=relation, current_type=relation.type, expected_type=expected_type), - model, - ) +class SetStrictWrongType(CompilationException): + def __init__(self, exc): + self.exc = exc + msg = str(self.exc) + super().__init__(msg=msg) -def package_not_found(package_name): - raise_dependency_error("Package {} was not found in the package index".format(package_name)) +class LoadAgateTableValueError(CompilationException): + def __init__(self, exc: ValueError, node): + self.exc = exc + self.node = node + msg = str(self.exc) + super().__init__(msg=msg) -def package_version_not_found( - package_name, version_range, available_versions, should_version_check -): - base_msg = ( - "Could not find a matching compatible version for package {}\n" - " Requested range: {}\n" - " Compatible versions: {}\n" - ) - addendum = ( - ( - "\n" - " Not shown: package versions incompatible with installed version of dbt-core\n" - " To include them, run 'dbt --no-version-check deps'" - ) - if should_version_check - else "" - ) - msg = base_msg.format(package_name, version_range, available_versions) + addendum - raise_dependency_error(msg) +class LoadAgateTableNotSeed(CompilationException): + def __init__(self, resource_type, node): + self.resource_type = resource_type + self.node = node + msg = f"can only load_agate_table for seeds (got a {self.resource_type})" + super().__init__(msg=msg) -def invalid_materialization_argument(name, argument): - raise_compiler_error( - "materialization '{}' received unknown argument '{}'.".format(name, argument) - ) +class MacrosSourcesUnWriteable(CompilationException): + def __init__(self, node): + self.node = node + msg = 'cannot "write" macros or sources' + super().__init__(msg=msg) -def system_error(operation_name): - raise_compiler_error( - "dbt encountered an error when attempting to {}. " - "If this error persists, please create an issue at: \n\n" - "https://github.com/dbt-labs/dbt-core".format(operation_name) - ) +class PackageNotInDeps(CompilationException): + def __init__(self, package_name: str, node): + self.package_name = package_name + self.node = node + msg = f"Node package named {self.package_name} not found!" + super().__init__(msg=msg) -class ConnectionException(Exception): - """ - There was a problem with the connection that returned a bad response, - timed out, or resulted in a file that is corrupt. - """ +class OperationsCannotRefEphemeralNodes(CompilationException): + def __init__(self, target_name: str, node): + self.target_name = target_name + self.node = node + msg = f"Operations can not ref() ephemeral nodes, but {target_name} is ephemeral" + super().__init__(msg=msg) - pass +class InvalidPersistDocsValueType(CompilationException): + def __init__(self, persist_docs: Any): + self.persist_docs = persist_docs + msg = ( + "Invalid value provided for 'persist_docs'. Expected dict " + f"but received {type(self.persist_docs)}" + ) + super().__init__(msg=msg) -def raise_dep_not_found(node, node_description, required_pkg): - raise_compiler_error( - 'Error while parsing {}.\nThe required package "{}" was not found. ' - "Is the package installed?\nHint: You may need to run " - "`dbt deps`.".format(node_description, required_pkg), - node=node, - ) +class InvalidInlineModelConfig(CompilationException): + def __init__(self, node): + self.node = node + msg = "Invalid inline model config" + super().__init__(msg=msg) -def multiple_matching_relations(kwargs, matches): - raise_compiler_error( - "get_relation returned more than one relation with the given args. " - "Please specify a database or schema to narrow down the result set." - "\n{}\n\n{}".format(kwargs, matches) - ) +class ConflictingConfigKeys(CompilationException): + def __init__(self, oldkey: str, newkey: str, node): + self.oldkey = oldkey + self.newkey = newkey + self.node = node + msg = f'Invalid config, has conflicting keys "{self.oldkey}" and "{self.newkey}"' + super().__init__(msg=msg) -def get_relation_returned_multiple_results(kwargs, matches): - multiple_matching_relations(kwargs, matches) +class InvalidNumberSourceArgs(CompilationException): + def __init__(self, args, node): + self.args = args + self.node = node + msg = f"source() takes exactly two arguments ({len(self.args)} given)" + super().__init__(msg=msg) -def approximate_relation_match(target, relation): - raise_compiler_error( - "When searching for a relation, dbt found an approximate match. " - "Instead of guessing \nwhich relation to use, dbt will move on. " - "Please delete {relation}, or rename it to be less ambiguous." - "\nSearched for: {target}\nFound: {relation}".format(target=target, relation=relation) - ) +class RequiredVarNotFound(CompilationException): + def __init__(self, var_name: str, merged: Dict, node): + self.var_name = var_name + self.merged = merged + self.node = node + super().__init__(msg=self.get_message()) -def raise_duplicate_macro_name(node_1, node_2, namespace) -> NoReturn: - duped_name = node_1.name - if node_1.package_name != node_2.package_name: - extra = ' ("{}" and "{}" are both in the "{}" namespace)'.format( - node_1.package_name, node_2.package_name, namespace - ) - else: - extra = "" - - raise_compiler_error( - 'dbt found two macros with the name "{}" in the namespace "{}"{}. ' - "Since these macros have the same name and exist in the same " - "namespace, dbt will be unable to decide which to call. To fix this, " - "change the name of one of these macros:\n- {} ({})\n- {} ({})".format( - duped_name, - namespace, - extra, - node_1.unique_id, - node_1.original_file_path, - node_2.unique_id, - node_2.original_file_path, - ) - ) + def get_message(self) -> str: + if self.node is not None: + node_name = self.node.name + else: + node_name = "" + dct = {k: self.merged[k] for k in self.merged} + pretty_vars = json.dumps(dct, sort_keys=True, indent=4) -def raise_duplicate_resource_name(node_1, node_2): - duped_name = node_1.name - node_type = NodeType(node_1.resource_type) - pluralized = ( - node_type.pluralize() - if node_1.resource_type == node_2.resource_type - else "resources" # still raise if ref() collision, e.g. model + seed - ) + msg = f"Required var '{self.var_name}' not found in config:\nVars supplied to {node_name} = {pretty_vars}" + return msg - action = "looking for" - # duplicate 'ref' targets - if node_type in NodeType.refable(): - formatted_name = f'ref("{duped_name}")' - # duplicate sources - elif node_type == NodeType.Source: - duped_name = node_1.get_full_source_name() - formatted_name = node_1.get_source_representation() - # duplicate docs blocks - elif node_type == NodeType.Documentation: - formatted_name = f'doc("{duped_name}")' - # duplicate generic tests - elif node_type == NodeType.Test and hasattr(node_1, "test_metadata"): - column_name = f'column "{node_1.column_name}" in ' if node_1.column_name else "" - model_name = node_1.file_key_name - duped_name = f'{node_1.name}" defined on {column_name}"{model_name}' - action = "running" - formatted_name = "tests" - # all other resource types - else: - formatted_name = duped_name - - # should this be raise_parsing_error instead? - raise_compiler_error( - f""" -dbt found two {pluralized} with the name "{duped_name}". -Since these resources have the same name, dbt will be unable to find the correct resource -when {action} {formatted_name}. +class PackageNotFoundForMacro(CompilationException): + def __init__(self, package_name: str): + self.package_name = package_name + msg = f"Could not find package '{self.package_name}'" + super().__init__(msg=msg) -To fix this, change the name of one of these resources: -- {node_1.unique_id} ({node_1.original_file_path}) -- {node_2.unique_id} ({node_2.original_file_path}) - """.strip() - ) +class DisallowSecretEnvVar(ParsingException): + def __init__(self, env_var_name: str): + self.env_var_name = env_var_name + super().__init__(msg=self.get_message()) -def raise_ambiguous_alias(node_1, node_2, duped_name=None): - if duped_name is None: - duped_name = f"{node_1.database}.{node_1.schema}.{node_1.alias}" - - raise_compiler_error( - 'dbt found two resources with the database representation "{}".\ndbt ' - "cannot create two resources with identical database representations. " - "To fix this,\nchange the configuration of one of these resources:" - "\n- {} ({})\n- {} ({})".format( - duped_name, - node_1.unique_id, - node_1.original_file_path, - node_2.unique_id, - node_2.original_file_path, + def get_message(self) -> str: + msg = ( + "Secret env vars are allowed only in profiles.yml or packages.yml. " + f"Found '{self.env_var_name}' referenced elsewhere." ) - ) + return msg -def raise_ambiguous_catalog_match(unique_id, match_1, match_2): - def get_match_string(match): - return "{}.{}".format( - match.get("metadata", {}).get("schema"), - match.get("metadata", {}).get("name"), - ) +class InvalidMacroArgType(CompilationException): + def __init__(self, method_name: str, arg_name: str, got_value: Any, expected_type): + self.method_name = method_name + self.arg_name = arg_name + self.got_value = got_value + self.expected_type = expected_type + super().__init__(msg=self.get_message()) - raise_compiler_error( - "dbt found two relations in your warehouse with similar database " - "identifiers. dbt\nis unable to determine which of these relations " - 'was created by the model "{unique_id}".\nIn order for dbt to ' - "correctly generate the catalog, one of the following relations must " - "be deleted or renamed:\n\n - {match_1_s}\n - {match_2_s}".format( - unique_id=unique_id, - match_1_s=get_match_string(match_1), - match_2_s=get_match_string(match_2), + def get_message(self) -> str: + got_type = type(self.got_value) + msg = ( + f"'adapter.{self.method_name}' expects argument " + f"'{self.arg_name}' to be of type '{self.expected_type}', instead got " + f"{self.got_value} ({got_type})" ) - ) - + return msg -def raise_patch_targets_not_found(patches): - patch_list = "\n\t".join( - "model {} (referenced in path {})".format(p.name, p.original_file_path) - for p in patches.values() - ) - raise_compiler_error( - "dbt could not find models for the following patches:\n\t{}".format(patch_list) - ) +class InvalidBoolean(CompilationException): + def __init__(self, return_value: Any, macro_name: str): + self.return_value = return_value + self.macro_name = macro_name + super().__init__(msg=self.get_message()) -def _fix_dupe_msg(path_1: str, path_2: str, name: str, type_name: str) -> str: - if path_1 == path_2: - return f"remove one of the {type_name} entries for {name} in this file:\n - {path_1!s}\n" - else: - return ( - f"remove the {type_name} entry for {name} in one of these files:\n" - f" - {path_1!s}\n{path_2!s}" + def get_message(self) -> str: + msg = ( + f"Macro '{self.macro_name}' returns '{self.return_value}'. It is not type 'bool' " + "and cannot not be converted reliably to a bool." ) + return msg -def raise_duplicate_patch_name(patch_1, existing_patch_path): - name = patch_1.name - fix = _fix_dupe_msg( - patch_1.original_file_path, - existing_patch_path, - name, - "resource", - ) - raise_compiler_error( - f"dbt found two schema.yml entries for the same resource named " - f"{name}. Resources and their associated columns may only be " - f"described a single time. To fix this, {fix}" - ) +class RefInvalidArgs(CompilationException): + def __init__(self, node, args): + self.node = node + self.args = args + super().__init__(msg=self.get_message()) + def get_message(self) -> str: + msg = f"ref() takes at most two arguments ({len(self.args)} given)" + return msg + + +class MetricInvalidArgs(CompilationException): + def __init__(self, node, args): + self.node = node + self.args = args + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"metric() takes at most two arguments ({len(self.args)} given)" + return msg + + +class RefBadContext(CompilationException): + def __init__(self, node, args): + self.node = node + self.args = args + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + # This explicitly references model['name'], instead of model['alias'], for + # better error messages. Ex. If models foo_users and bar_users are aliased + # to 'users', in their respective schemas, then you would want to see + # 'bar_users' in your error messge instead of just 'users'. + if isinstance(self.node, dict): + model_name = self.node["name"] + else: + model_name = self.node.name + + ref_args = ", ".join("'{}'".format(a) for a in self.args) + ref_string = f"{{{{ ref({ref_args}) }}}}" + + msg = f"""dbt was unable to infer all dependencies for the model "{model_name}". +This typically happens when ref() is placed within a conditional block. + +To fix this, add the following hint to the top of the model "{model_name}": + +-- depends_on: {ref_string}""" + + return msg + + +class InvalidDocArgs(CompilationException): + def __init__(self, node, args): + self.node = node + self.args = args + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"doc() takes at most two arguments ({len(self.args)} given)" + return msg + + +class DocTargetNotFound(CompilationException): + def __init__(self, node, target_doc_name: str, target_doc_package: Optional[str]): + self.node = node + self.target_doc_name = target_doc_name + self.target_doc_package = target_doc_package + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + target_package_string = "" + if self.target_doc_package is not None: + target_package_string = f"in package '{self. target_doc_package}' " + msg = f"Documentation for '{self.node.unique_id}' depends on doc '{self.target_doc_name}' {target_package_string} which was not found" + return msg + + +class MacroInvalidDispatchArg(CompilationException): + def __init__(self, macro_name: str): + self.macro_name = macro_name + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"""\ + The "packages" argument of adapter.dispatch() has been deprecated. + Use the "macro_namespace" argument instead. + + Raised during dispatch for: {self.macro_name} + + For more information, see: + + https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch + """ + return msg + + +class DuplicateMacroName(CompilationException): + def __init__(self, node_1, node_2, namespace: str): + self.node_1 = node_1 + self.node_2 = node_2 + self.namespace = namespace + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + duped_name = self.node_1.name + if self.node_1.package_name != self.node_2.package_name: + extra = f' ("{self.node_1.package_name}" and "{self.node_2.package_name}" are both in the "{self.namespace}" namespace)' + else: + extra = "" + + msg = ( + f'dbt found two macros with the name "{duped_name}" in the namespace "{self.namespace}"{extra}. ' + "Since these macros have the same name and exist in the same " + "namespace, dbt will be unable to decide which to call. To fix this, " + f"change the name of one of these macros:\n- {self.node_1.unique_id} " + f"({self.node_1.original_file_path})\n- {self.node_2.unique_id} ({self.node_2.original_file_path})" + ) + + return msg + + +# parser level exceptions +class InvalidDictParse(ParsingException): + def __init__(self, exc: ValidationError, node): + self.exc = exc + self.node = node + msg = self.validator_error_message(exc) + super().__init__(msg=msg) + + +class InvalidConfigUpdate(ParsingException): + def __init__(self, exc: ValidationError, node): + self.exc = exc + self.node = node + msg = self.validator_error_message(exc) + super().__init__(msg=msg) + + +class PythonParsingException(ParsingException): + def __init__(self, exc: SyntaxError, node): + self.exc = exc + self.node = node + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + validated_exc = self.validator_error_message(self.exc) + msg = f"{validated_exc}\n{self.exc.text}" + return msg + + +class PythonLiteralEval(ParsingException): + def __init__(self, exc: Exception, node): + self.exc = exc + self.node = node + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"Error when trying to literal_eval an arg to dbt.ref(), dbt.source(), dbt.config() or dbt.config.get() \n{self.exc}\n" + "https://docs.python.org/3/library/ast.html#ast.literal_eval\n" + "In dbt python model, `dbt.ref`, `dbt.source`, `dbt.config`, `dbt.config.get` function args only support Python literal structures" + ) + + return msg + + +class InvalidModelConfig(ParsingException): + def __init__(self, exc: ValidationError, node): + self.msg = self.validator_error_message(exc) + self.node = node + super().__init__(msg=self.msg) + + +class YamlParseListFailure(ParsingException): + def __init__( + self, + path: str, + key: str, + yaml_data: List, + cause, + ): + self.path = path + self.key = key + self.yaml_data = yaml_data + self.cause = cause + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + if isinstance(self.cause, str): + reason = self.cause + elif isinstance(self.cause, ValidationError): + reason = self.validator_error_message(self.cause) + else: + reason = self.cause.msg + msg = f"Invalid {self.key} config given in {self.path} @ {self.key}: {self.yaml_data} - {reason}" + return msg + + +class YamlParseDictFailure(ParsingException): + def __init__( + self, + path: str, + key: str, + yaml_data: Dict[str, Any], + cause, + ): + self.path = path + self.key = key + self.yaml_data = yaml_data + self.cause = cause + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + if isinstance(self.cause, str): + reason = self.cause + elif isinstance(self.cause, ValidationError): + reason = self.validator_error_message(self.cause) + else: + reason = self.cause.msg + msg = f"Invalid {self.key} config given in {self.path} @ {self.key}: {self.yaml_data} - {reason}" + return msg + + +class YamlLoadFailure(ParsingException): + def __init__(self, project_name: Optional[str], path: str, exc: ValidationException): + self.project_name = project_name + self.path = path + self.exc = exc + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + reason = self.validator_error_message(self.exc) + + msg = f"Error reading {self.project_name}: {self.path} - {reason}" + + return msg + + +class InvalidTestConfig(ParsingException): + def __init__(self, exc: ValidationError, node): + self.msg = self.validator_error_message(exc) + self.node = node + super().__init__(msg=self.msg) + + +class InvalidSchemaConfig(ParsingException): + def __init__(self, exc: ValidationError, node): + self.msg = self.validator_error_message(exc) + self.node = node + super().__init__(msg=self.msg) + + +class InvalidSnapshopConfig(ParsingException): + def __init__(self, exc: ValidationError, node): + self.msg = self.validator_error_message(exc) + self.node = node + super().__init__(msg=self.msg) + + +class SameKeyNested(CompilationException): + def __init__(self): + msg = "Test cannot have the same key at the top-level and in config" + super().__init__(msg=msg) + + +class TestArgIncludesModel(CompilationException): + def __init__(self): + msg = 'Test arguments include "model", which is a reserved argument' + super().__init__(msg=msg) + + +class UnexpectedTestNamePattern(CompilationException): + def __init__(self, test_name: str): + self.test_name = test_name + msg = f"Test name string did not match expected pattern: {self.test_name}" + super().__init__(msg=msg) + + +class CustomMacroPopulatingConfigValues(CompilationException): + def __init__( + self, target_name: str, column_name: Optional[str], name: str, key: str, err_msg: str + ): + self.target_name = target_name + self.column_name = column_name + self.name = name + self.key = key + self.err_msg = err_msg + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + # Generic tests do not include custom macros in the Jinja + # rendering context, so this will almost always fail. As it + # currently stands, the error message is inscrutable, which + # has caused issues for some projects migrating from + # pre-0.20.0 to post-0.20.0. + # See https://github.com/dbt-labs/dbt-core/issues/4103 + # and https://github.com/dbt-labs/dbt-core/issues/5294 + + msg = ( + f"The {self.target_name}.{self.column_name} column's " + f'"{self.name}" test references an undefined ' + f"macro in its {self.key} configuration argument. " + f"The macro {self.err_msg}.\n" + "Please note that the generic test configuration parser " + "currently does not support using custom macros to " + "populate configuration values" + ) + return msg + + +class TagsNotListOfStrings(CompilationException): + def __init__(self, tags: Any): + self.tags = tags + msg = f"got {self.tags} ({type(self.tags)}) for tags, expected a list of strings" + super().__init__(msg=msg) + + +class TagNotString(CompilationException): + def __init__(self, tag: Any): + self.tag = tag + msg = f"got {self.tag} ({type(self.tag)}) for tag, expected a str" + super().__init__(msg=msg) + + +class TestNameNotString(ParsingException): + def __init__(self, test_name: Any): + self.test_name = test_name + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + + msg = f"test name must be a str, got {type(self.test_name)} (value {self.test_name})" + return msg + + +class TestArgsNotDict(ParsingException): + def __init__(self, test_args: Any): + self.test_args = test_args + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + + msg = f"test arguments must be a dict, got {type(self.test_args)} (value {self.test_args})" + return msg + + +class TestDefinitionDictLength(ParsingException): + def __init__(self, test): + self.test = test + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + + msg = ( + "test definition dictionary must have exactly one key, got" + f" {self.test} instead ({len(self.test)} keys)" + ) + return msg + + +class TestInvalidType(ParsingException): + def __init__(self, test: Any): + self.test = test + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"test must be dict or str, got {type(self.test)} (value {self.test})" + return msg + + +# This is triggered across multiple files +class EnvVarMissing(ParsingException): + def __init__(self, var: str): + self.var = var + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"Env var required but not provided: '{self.var}'" + return msg + + +class TargetNotFound(CompilationException): + def __init__( + self, + node, + target_name: str, + target_kind: str, + target_package: Optional[str] = None, + disabled: Optional[bool] = None, + ): + self.node = node + self.target_name = target_name + self.target_kind = target_kind + self.target_package = target_package + self.disabled = disabled + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + original_file_path = self.node.original_file_path + unique_id = self.node.unique_id + resource_type_title = self.node.resource_type.title() + + if self.disabled is None: + reason = "was not found or is disabled" + elif self.disabled is True: + reason = "is disabled" + else: + reason = "was not found" + + target_package_string = "" + if self.target_package is not None: + target_package_string = f"in package '{self.target_package}' " + + msg = ( + f"{resource_type_title} '{unique_id}' ({original_file_path}) depends on a " + f"{self.target_kind} named '{self.target_name}' {target_package_string}which {reason}" + ) + return msg + + +class DuplicateSourcePatchName(CompilationException): + def __init__(self, patch_1, patch_2): + self.patch_1 = patch_1 + self.patch_2 = patch_2 + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + name = f"{self.patch_1.overrides}.{self.patch_1.name}" + fix = self._fix_dupe_msg( + self.patch_1.path, + self.patch_2.path, + name, + "sources", + ) + msg = ( + f"dbt found two schema.yml entries for the same source named " + f"{self.patch_1.name} in package {self.patch_1.overrides}. Sources may only be " + f"overridden a single time. To fix this, {fix}" + ) + return msg + + +class DuplicateMacroPatchName(CompilationException): + def __init__(self, patch_1, existing_patch_path): + self.patch_1 = patch_1 + self.existing_patch_path = existing_patch_path + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + package_name = self.patch_1.package_name + name = self.patch_1.name + fix = self._fix_dupe_msg( + self.patch_1.original_file_path, self.existing_patch_path, name, "macros" + ) + msg = ( + f"dbt found two schema.yml entries for the same macro in package " + f"{package_name} named {name}. Macros may only be described a single " + f"time. To fix this, {fix}" + ) + return msg + + +# core level exceptions +class DuplicateAlias(AliasException): + def __init__(self, kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str): + self.kwargs = kwargs + self.aliases = aliases + self.canonical_key = canonical_key + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + # dupe found: go through the dict so we can have a nice-ish error + key_names = ", ".join( + "{}".format(k) for k in self.kwargs if self.aliases.get(k) == self.canonical_key + ) + msg = f'Got duplicate keys: ({key_names}) all map to "{self.canonical_key}"' + return msg + + +# Postgres Exceptions + + +class UnexpectedDbReference(NotImplementedException): + def __init__(self, adapter, database, expected): + self.adapter = adapter + self.database = database + self.expected = expected + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"Cross-db references not allowed in {self.adapter} ({self.database} vs {self.expected})" + return msg + + +class CrossDbReferenceProhibited(CompilationException): + def __init__(self, adapter, exc_msg: str): + self.adapter = adapter + self.exc_msg = exc_msg + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"Cross-db references not allowed in adapter {self.adapter}: Got {self.exc_msg}" + return msg + + +class IndexConfigNotDict(CompilationException): + def __init__(self, raw_index: Any): + self.raw_index = raw_index + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"Invalid index config:\n" + f" Got: {self.raw_index}\n" + f' Expected a dictionary with at minimum a "columns" key' + ) + return msg + + +class InvalidIndexConfig(CompilationException): + def __init__(self, exc: TypeError): + self.exc = exc + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + validator_msg = self.validator_error_message(self.exc) + msg = f"Could not parse index config: {validator_msg}" + return msg + + +# adapters exceptions +class InvalidMacroResult(CompilationException): + def __init__(self, freshness_macro_name: str, table): + self.freshness_macro_name = freshness_macro_name + self.table = table + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f'Got an invalid result from "{self.freshness_macro_name}" macro: {[tuple(r) for r in self.table]}' + + return msg + + +class SnapshotTargetNotSnapshotTable(CompilationException): + def __init__(self, missing: List): + self.missing = missing + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = 'Snapshot target is not a snapshot table (missing "{}")'.format( + '", "'.join(self.missing) + ) + return msg + + +class SnapshotTargetIncomplete(CompilationException): + def __init__(self, extra: List, missing: List): + self.extra = extra + self.missing = missing + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + 'Snapshot target has ("{}") but not ("{}") - is it an ' + "unmigrated previous version archive?".format( + '", "'.join(self.extra), '", "'.join(self.missing) + ) + ) + return msg + + +class RenameToNoneAttempted(CompilationException): + def __init__(self, src_name: str, dst_name: str, name: str): + self.src_name = src_name + self.dst_name = dst_name + self.name = name + self.msg = f"Attempted to rename {self.src_name} to {self.dst_name} for {self.name}" + super().__init__(msg=self.msg) + + +class NullRelationDropAttempted(CompilationException): + def __init__(self, name: str): + self.name = name + self.msg = f"Attempted to drop a null relation for {self.name}" + super().__init__(msg=self.msg) + + +class NullRelationCacheAttempted(CompilationException): + def __init__(self, name: str): + self.name = name + self.msg = f"Attempted to cache a null relation for {self.name}" + super().__init__(msg=self.msg) + + +class InvalidQuoteConfigType(CompilationException): + def __init__(self, quote_config: Any): + self.quote_config = quote_config + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + 'The seed configuration value of "quote_columns" has an ' + f"invalid type {type(self.quote_config)}" + ) + return msg + + +class MultipleDatabasesNotAllowed(CompilationException): + def __init__(self, databases): + self.databases = databases + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = str(self.databases) + return msg + + +class RelationTypeNull(CompilationException): + def __init__(self, relation): + self.relation = relation + self.msg = f"Tried to drop relation {self.relation}, but its type is null." + super().__init__(msg=self.msg) + + +class MaterializationNotAvailable(CompilationException): + def __init__(self, model, adapter_type: str): + self.model = model + self.adapter_type = adapter_type + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + materialization = self.model.get_materialization() + msg = f"Materialization '{materialization}' is not available for {self.adapter_type}!" + return msg + + +class RelationReturnedMultipleResults(CompilationException): + def __init__(self, kwargs: Mapping[str, Any], matches: List): + self.kwargs = kwargs + self.matches = matches + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + "get_relation returned more than one relation with the given args. " + "Please specify a database or schema to narrow down the result set." + f"\n{self.kwargs}\n\n{self.matches}" + ) + return msg + + +class ApproximateMatch(CompilationException): + def __init__(self, target, relation): + self.target = target + self.relation = relation + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + + msg = ( + "When searching for a relation, dbt found an approximate match. " + "Instead of guessing \nwhich relation to use, dbt will move on. " + f"Please delete {self.relation}, or rename it to be less ambiguous." + f"\nSearched for: {self.target}\nFound: {self.relation}" + ) + + return msg + + +# adapters exceptions +class UnexpectedNull(DatabaseException): + def __init__(self, field_name: str, source): + self.field_name = field_name + self.source = source + msg = ( + f"Expected a non-null value when querying field '{self.field_name}' of table " + f" {self.source} but received value 'null' instead" + ) + super().__init__(msg) + + +class UnexpectedNonTimestamp(DatabaseException): + def __init__(self, field_name: str, source, dt: Any): + self.field_name = field_name + self.source = source + self.type_name = type(dt).__name__ + msg = ( + f"Expected a timestamp value when querying field '{self.field_name}' of table " + f"{self.source} but received value of type '{self.type_name}' instead" + ) + super().__init__(msg) + + +# deps exceptions +class MultipleVersionGitDeps(DependencyException): + def __init__(self, git: str, requested): + self.git = git + self.requested = requested + msg = ( + "git dependencies should contain exactly one version. " + f"{self.git} contains: {self.requested}" + ) + super().__init__(msg) + + +class DuplicateProjectDependency(DependencyException): + def __init__(self, project_name: str): + self.project_name = project_name + msg = ( + f'Found duplicate project "{self.project_name}". This occurs when ' + "a dependency has the same project name as some other dependency." + ) + super().__init__(msg) + + +class DuplicateDependencyToRoot(DependencyException): + def __init__(self, project_name: str): + self.project_name = project_name + msg = ( + "Found a dependency with the same name as the root project " + f'"{self.project_name}". Package names must be unique in a project.' + " Please rename one of these packages." + ) + super().__init__(msg) + + +class MismatchedDependencyTypes(DependencyException): + def __init__(self, new, old): + self.new = new + self.old = old + msg = ( + f"Cannot incorporate {self.new} ({self.new.__class__.__name__}) in {self.old} " + f"({self.old.__class__.__name__}): mismatched types" + ) + super().__init__(msg) + + +class PackageVersionNotFound(DependencyException): + def __init__( + self, + package_name: str, + version_range, + available_versions: List[str], + should_version_check: bool, + ): + self.package_name = package_name + self.version_range = version_range + self.available_versions = available_versions + self.should_version_check = should_version_check + super().__init__(self.get_message()) + + def get_message(self) -> str: + base_msg = ( + "Could not find a matching compatible version for package {}\n" + " Requested range: {}\n" + " Compatible versions: {}\n" + ) + addendum = ( + ( + "\n" + " Not shown: package versions incompatible with installed version of dbt-core\n" + " To include them, run 'dbt --no-version-check deps'" + ) + if self.should_version_check + else "" + ) + msg = ( + base_msg.format(self.package_name, self.version_range, self.available_versions) + + addendum + ) + return msg + + +class PackageNotFound(DependencyException): + def __init__(self, package_name: str): + self.package_name = package_name + msg = f"Package {self.package_name} was not found in the package index" + super().__init__(msg) + + +# config level exceptions + + +class ProfileConfigInvalid(DbtProfileError): + def __init__(self, exc: ValidationError): + self.exc = exc + msg = self.validator_error_message(self.exc) + super().__init__(msg=msg) + + +class ProjectContractInvalid(DbtProjectError): + def __init__(self, exc: ValidationError): + self.exc = exc + msg = self.validator_error_message(self.exc) + super().__init__(msg=msg) + + +class ProjectContractBroken(DbtProjectError): + def __init__(self, exc: ValidationError): + self.exc = exc + msg = self.validator_error_message(self.exc) + super().__init__(msg=msg) + + +class ConfigContractBroken(DbtProjectError): + def __init__(self, exc: ValidationError): + self.exc = exc + msg = self.validator_error_message(self.exc) + super().__init__(msg=msg) + + +class NonUniquePackageName(CompilationException): + def __init__(self, project_name: str): + self.project_name = project_name + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + "dbt found more than one package with the name " + f'"{self.project_name}" included in this project. Package ' + "names must be unique in a project. Please rename " + "one of these packages." + ) + return msg + + +class UninstalledPackagesFound(CompilationException): + def __init__( + self, + count_packages_specified: int, + count_packages_installed: int, + packages_install_path: str, + ): + self.count_packages_specified = count_packages_specified + self.count_packages_installed = count_packages_installed + self.packages_install_path = packages_install_path + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"dbt found {self.count_packages_specified} package(s) " + "specified in packages.yml, but only " + f"{self.count_packages_installed} package(s) installed " + f'in {self.packages_install_path}. Run "dbt deps" to ' + "install package dependencies." + ) + return msg + + +class VarsArgNotYamlDict(CompilationException): + def __init__(self, var_type): + self.var_type = var_type + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + type_name = self.var_type.__name__ + + msg = f"The --vars argument must be a YAML dictionary, but was of type '{type_name}'" + return msg + + +# contracts level + + +class DuplicateMacroInPackage(CompilationException): + def __init__(self, macro, macro_mapping: Mapping): + self.macro = macro + self.macro_mapping = macro_mapping + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + other_path = self.macro_mapping[self.macro.unique_id].original_file_path + # subtract 2 for the "Compilation Error" indent + # note that the line wrap eats newlines, so if you want newlines, + # this is the result :( + msg = line_wrap_message( + f"""\ + dbt found two macros named "{self.macro.name}" in the project + "{self.macro.package_name}". + + + To fix this error, rename or remove one of the following + macros: + + - {self.macro.original_file_path} + + - {other_path} + """, + subtract=2, + ) + return msg + + +class DuplicateMaterializationName(CompilationException): + def __init__(self, macro, other_macro): + self.macro = macro + self.other_macro = other_macro + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + macro_name = self.macro.name + macro_package_name = self.macro.package_name + other_package_name = self.other_macro.macro.package_name + + msg = ( + f"Found two materializations with the name {macro_name} (packages " + f"{macro_package_name} and {other_package_name}). dbt cannot resolve " + "this ambiguity" + ) + return msg + + +# jinja exceptions +class MissingConfig(CompilationException): + def __init__(self, unique_id: str, name: str): + self.unique_id = unique_id + self.name = name + msg = ( + f"Model '{self.unique_id}' does not define a required config parameter '{self.name}'." + ) + super().__init__(msg=msg) + + +class MissingMaterialization(CompilationException): + def __init__(self, model, adapter_type): + self.model = model + self.adapter_type = adapter_type + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + materialization = self.model.get_materialization() + + valid_types = "'default'" + + if self.adapter_type != "default": + valid_types = f"'default' and '{self.adapter_type}'" + + msg = f"No materialization '{materialization}' was found for adapter {self.adapter_type}! (searched types {valid_types})" + return msg + + +class MissingRelation(CompilationException): + def __init__(self, relation, model=None): + self.relation = relation + self.model = model + msg = f"Relation {self.relation} not found!" + super().__init__(msg=msg) -def raise_duplicate_macro_patch_name(patch_1, existing_patch_path): - package_name = patch_1.package_name - name = patch_1.name - fix = _fix_dupe_msg(patch_1.original_file_path, existing_patch_path, name, "macros") - raise_compiler_error( - f"dbt found two schema.yml entries for the same macro in package " - f"{package_name} named {name}. Macros may only be described a single " - f"time. To fix this, {fix}" - ) +class AmbiguousAlias(CompilationException): + def __init__(self, node_1, node_2, duped_name=None): + self.node_1 = node_1 + self.node_2 = node_2 + if duped_name is None: + self.duped_name = f"{self.node_1.database}.{self.node_1.schema}.{self.node_1.alias}" + else: + self.duped_name = duped_name + super().__init__(msg=self.get_message()) -def raise_duplicate_source_patch_name(patch_1, patch_2): - name = f"{patch_1.overrides}.{patch_1.name}" - fix = _fix_dupe_msg( - patch_1.path, - patch_2.path, - name, - "sources", - ) - raise_compiler_error( - f"dbt found two schema.yml entries for the same source named " - f"{patch_1.name} in package {patch_1.overrides}. Sources may only be " - f"overridden a single time. To fix this, {fix}" - ) + def get_message(self) -> str: + + msg = ( + f'dbt found two resources with the database representation "{self.duped_name}".\ndbt ' + "cannot create two resources with identical database representations. " + "To fix this,\nchange the configuration of one of these resources:" + f"\n- {self.node_1.unique_id} ({self.node_1.original_file_path})\n- {self.node_2.unique_id} ({self.node_2.original_file_path})" + ) + return msg -def raise_invalid_property_yml_version(path, issue): - raise_compiler_error( - "The yml property file at {} is invalid because {}. Please consult the " - "documentation for more information on yml property file syntax:\n\n" - "https://docs.getdbt.com/reference/configs-and-properties".format(path, issue) - ) +class AmbiguousCatalogMatch(CompilationException): + def __init__(self, unique_id: str, match_1, match_2): + self.unique_id = unique_id + self.match_1 = match_1 + self.match_2 = match_2 + super().__init__(msg=self.get_message()) + def get_match_string(self, match): + match_schema = match.get("metadata", {}).get("schema") + match_name = match.get("metadata", {}).get("name") + return f"{match_schema}.{match_name}" -def raise_unrecognized_credentials_type(typename, supported_types): - raise_compiler_error( - 'Unrecognized credentials type "{}" - supported types are ({})'.format( - typename, ", ".join('"{}"'.format(t) for t in supported_types) + def get_message(self) -> str: + msg = ( + "dbt found two relations in your warehouse with similar database identifiers. " + "dbt\nis unable to determine which of these relations was created by the model " + f'"{self.unique_id}".\nIn order for dbt to correctly generate the catalog, one ' + "of the following relations must be deleted or renamed:\n\n - " + f"{self.get_match_string(self.match_1)}\n - {self.get_match_string(self.match_2)}" ) - ) + return msg + + +class CacheInconsistency(InternalException): + def __init__(self, msg: str): + self.msg = msg + formatted_msg = f"Cache inconsistency detected: {self.msg}" + super().__init__(msg=formatted_msg) + + +class NewNameAlreadyInCache(CacheInconsistency): + def __init__(self, old_key: str, new_key: str): + self.old_key = old_key + self.new_key = new_key + msg = ( + f'in rename of "{self.old_key}" -> "{self.new_key}", new name is in the cache already' + ) + super().__init__(msg) + + +class ReferencedLinkNotCached(CacheInconsistency): + def __init__(self, referenced_key: str): + self.referenced_key = referenced_key + msg = f"in add_link, referenced link key {self.referenced_key} not in cache!" + super().__init__(msg) + + +class DependentLinkNotCached(CacheInconsistency): + def __init__(self, dependent_key: str): + self.dependent_key = dependent_key + msg = f"in add_link, dependent link key {self.dependent_key} not in cache!" + super().__init__(msg) + + +class TruncatedModelNameCausedCollision(CacheInconsistency): + def __init__(self, new_key, relations: Dict): + self.new_key = new_key + self.relations = relations + super().__init__(self.get_message()) + + def get_message(self) -> str: + # Tell user when collision caused by model names truncated during + # materialization. + match = re.search("__dbt_backup|__dbt_tmp$", self.new_key.identifier) + if match: + truncated_model_name_prefix = self.new_key.identifier[: match.start()] + message_addendum = ( + "\n\nName collisions can occur when the length of two " + "models' names approach your database's builtin limit. " + "Try restructuring your project such that no two models " + f"share the prefix '{truncated_model_name_prefix}'. " + "Then, clean your warehouse of any removed models." + ) + else: + message_addendum = "" + + msg = f"in rename, new key {self.new_key} already in cache: {list(self.relations.keys())}{message_addendum}" + + return msg + + +class NoneRelationFound(CacheInconsistency): + def __init__(self): + msg = "in get_relations, a None relation was found in the cache!" + super().__init__(msg) + + +# this is part of the context and also raised in dbt.contracts.relation.py +class DataclassNotDict(CompilationException): + def __init__(self, obj: Any): + self.obj = obj + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f'The object ("{self.obj}") was used as a dictionary. This ' + "capability has been removed from objects of this type." + ) + + return msg + + +class DependencyNotFound(CompilationException): + def __init__(self, node, node_description, required_pkg): + self.node = node + self.node_description = node_description + self.required_pkg = required_pkg + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"Error while parsing {self.node_description}.\nThe required package " + f'"{self.required_pkg}" was not found. Is the package installed?\n' + "Hint: You may need to run `dbt deps`." + ) + + return msg + + +class DuplicatePatchPath(CompilationException): + def __init__(self, patch_1, existing_patch_path): + self.patch_1 = patch_1 + self.existing_patch_path = existing_patch_path + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + name = self.patch_1.name + fix = self._fix_dupe_msg( + self.patch_1.original_file_path, + self.existing_patch_path, + name, + "resource", + ) + msg = ( + f"dbt found two schema.yml entries for the same resource named " + f"{name}. Resources and their associated columns may only be " + f"described a single time. To fix this, {fix}" + ) + return msg + + +# should this inherit ParsingException instead? +class DuplicateResourceName(CompilationException): + def __init__(self, node_1, node_2): + self.node_1 = node_1 + self.node_2 = node_2 + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + duped_name = self.node_1.name + node_type = NodeType(self.node_1.resource_type) + pluralized = ( + node_type.pluralize() + if self.node_1.resource_type == self.node_2.resource_type + else "resources" # still raise if ref() collision, e.g. model + seed + ) + + action = "looking for" + # duplicate 'ref' targets + if node_type in NodeType.refable(): + formatted_name = f'ref("{duped_name}")' + # duplicate sources + elif node_type == NodeType.Source: + duped_name = self.node_1.get_full_source_name() + formatted_name = self.node_1.get_source_representation() + # duplicate docs blocks + elif node_type == NodeType.Documentation: + formatted_name = f'doc("{duped_name}")' + # duplicate generic tests + elif node_type == NodeType.Test and hasattr(self.node_1, "test_metadata"): + column_name = ( + f'column "{self.node_1.column_name}" in ' if self.node_1.column_name else "" + ) + model_name = self.node_1.file_key_name + duped_name = f'{self.node_1.name}" defined on {column_name}"{model_name}' + action = "running" + formatted_name = "tests" + # all other resource types + else: + formatted_name = duped_name + + msg = f""" +dbt found two {pluralized} with the name "{duped_name}". + +Since these resources have the same name, dbt will be unable to find the correct resource +when {action} {formatted_name}. + +To fix this, change the name of one of these resources: +- {self.node_1.unique_id} ({self.node_1.original_file_path}) +- {self.node_2.unique_id} ({self.node_2.original_file_path}) + """.strip() + return msg + + +class InvalidPropertyYML(CompilationException): + def __init__(self, path: str, issue: str): + self.path = path + self.issue = issue + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"The yml property file at {self.path} is invalid because {self.issue}. " + "Please consult the documentation for more information on yml property file " + "syntax:\n\nhttps://docs.getdbt.com/reference/configs-and-properties" + ) + return msg + + +class PropertyYMLMissingVersion(InvalidPropertyYML): + def __init__(self, path: str): + self.path = path + self.issue = f"the yml property file {self.path} is missing a version tag" + super().__init__(self.path, self.issue) + + +class PropertyYMLVersionNotInt(InvalidPropertyYML): + def __init__(self, path: str, version: Any): + self.path = path + self.version = version + self.issue = ( + "its 'version:' tag must be an integer (e.g. version: 2)." + f" {self.version} is not an integer" + ) + super().__init__(self.path, self.issue) + + +class PropertyYMLInvalidTag(InvalidPropertyYML): + def __init__(self, path: str, version: int): + self.path = path + self.version = version + self.issue = f"its 'version:' tag is set to {self.version}. Only 2 is supported" + super().__init__(self.path, self.issue) + + +class RelationWrongType(CompilationException): + def __init__(self, relation, expected_type, model=None): + self.relation = relation + self.expected_type = expected_type + self.model = model + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"Trying to create {self.expected_type} {self.relation}, " + f"but it currently exists as a {self.relation.type}. Either " + f"drop {self.relation} manually, or run dbt with " + "`--full-refresh` and dbt will drop it for you." + ) + + return msg + + +# These are copies of what's in dbt/context/exceptions_jinja.py to not immediately break adapters +# utilizing these functions as exceptions. These are direct copies to avoid circular imports. +# They will be removed in 1 (or 2?) versions. Issue to be created to ensure it happens. + +# TODO: add deprecation to functions +def warn(msg, node=None): + warn_or_error(JinjaLogWarning(msg=msg, node_info=get_node_info())) + return "" + + +def missing_config(model, name) -> NoReturn: + raise MissingConfig(unique_id=model.unique_id, name=name) + + +def missing_materialization(model, adapter_type) -> NoReturn: + raise MissingMaterialization(model=model, adapter_type=adapter_type) + + +def missing_relation(relation, model=None) -> NoReturn: + raise MissingRelation(relation, model) + + +def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn: + raise AmbiguousAlias(node_1, node_2, duped_name) + + +def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: + raise AmbiguousCatalogMatch(unique_id, match_1, match_2) + + +def raise_cache_inconsistent(message) -> NoReturn: + raise CacheInconsistency(message) -def raise_not_implemented(msg): - raise NotImplementedException("ERROR: {}".format(msg)) +def raise_dataclass_not_dict(obj) -> NoReturn: + raise DataclassNotDict(obj) + +# note: this is called all over the code in addition to in jinja +def raise_compiler_error(msg, node=None) -> NoReturn: + raise CompilationException(msg, node) + + +def raise_database_error(msg, node=None) -> NoReturn: + raise DatabaseException(msg, node) + + +def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn: + raise DependencyNotFound(node, node_description, required_pkg) + + +def raise_dependency_error(msg) -> NoReturn: + raise DependencyException(scrub_secrets(msg, env_secrets())) + + +def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn: + raise DuplicatePatchPath(patch_1, existing_patch_path) + + +def raise_duplicate_resource_name(node_1, node_2) -> NoReturn: + raise DuplicateResourceName(node_1, node_2) + + +def raise_invalid_property_yml_version(path, issue) -> NoReturn: + raise InvalidPropertyYML(path, issue) + + +def raise_not_implemented(msg) -> NoReturn: + raise NotImplementedException(msg) + + +def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: + raise RelationWrongType(relation, expected_type, model) + + +# these were implemented in core so deprecating here by calling the new exception directly def raise_duplicate_alias( kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str ) -> NoReturn: - # dupe found: go through the dict so we can have a nice-ish error - key_names = ", ".join("{}".format(k) for k in kwargs if aliases.get(k) == canonical_key) + raise DuplicateAlias(kwargs, aliases, canonical_key) + - raise AliasException(f'Got duplicate keys: ({key_names}) all map to "{canonical_key}"') +def raise_duplicate_source_patch_name(patch_1, patch_2): + raise DuplicateSourcePatchName(patch_1, patch_2) -def warn(msg, node=None): - dbt.events.functions.warn_or_error( - JinjaLogWarning(msg=msg, node_info=get_node_info()), +def raise_duplicate_macro_patch_name(patch_1, existing_patch_path): + raise DuplicateMacroPatchName(patch_1, existing_patch_path) + + +def raise_duplicate_macro_name(node_1, node_2, namespace) -> NoReturn: + raise DuplicateMacroName(node_1, node_2, namespace) + + +def approximate_relation_match(target, relation): + raise ApproximateMatch(target, relation) + + +def get_relation_returned_multiple_results(kwargs, matches): + raise RelationReturnedMultipleResults(kwargs, matches) + + +def system_error(operation_name): + # Note: This was converted for core to use SymbolicLinkError because it's the only way it was used. Maintaining flexibility here for now. + msg = ( + f"dbt encountered an error when attempting to {operation_name}. " + "If this error persists, please create an issue at: \n\n" + "https://github.com/dbt-labs/dbt-core" + ) + raise CompilationException(msg) + + +def invalid_materialization_argument(name, argument): + raise InvalidMaterializationArg(name, argument) + + +def bad_package_spec(repo, spec, error_message): + msg = f"Error checking out spec='{spec}' for repo {repo}\n{error_message}" + raise InternalException(scrub_secrets(msg, env_secrets())) + + +def raise_git_cloning_error(error: CommandResultError) -> NoReturn: + error.cmd = list(scrub_secrets(str(error.cmd), env_secrets())) + raise error + + +def raise_git_cloning_problem(repo) -> NoReturn: + raise GitCloningProblem(repo) + + +def macro_invalid_dispatch_arg(macro_name) -> NoReturn: + raise MacroInvalidDispatchArg(macro_name) + + +def dependency_not_found(node, dependency): + raise GraphDependencyNotFound(node, dependency) + + +def target_not_found( + node, + target_name: str, + target_kind: str, + target_package: Optional[str] = None, + disabled: Optional[bool] = None, +) -> NoReturn: + raise TargetNotFound( node=node, + target_name=target_name, + target_kind=target_kind, + target_package=target_package, + disabled=disabled, ) - return "" -# Update this when a new function should be added to the -# dbt context's `exceptions` key! -CONTEXT_EXPORTS = { - fn.__name__: fn - for fn in [ - warn, - missing_config, - missing_materialization, - missing_relation, - raise_ambiguous_alias, - raise_ambiguous_catalog_match, - raise_cache_inconsistent, - raise_dataclass_not_dict, - raise_compiler_error, - raise_database_error, - raise_dep_not_found, - raise_dependency_error, - raise_duplicate_patch_name, - raise_duplicate_resource_name, - raise_invalid_property_yml_version, - raise_not_implemented, - relation_wrong_type, - ] -} - - -def wrapper(model): - def wrap(func): - @functools.wraps(func) - def inner(*args, **kwargs): - try: - return func(*args, **kwargs) - except RuntimeException as exc: - exc.add_node(model) - raise exc - - return inner - - return wrap - - -def wrapped_exports(model): - wrap = wrapper(model) - return {name: wrap(export) for name, export in CONTEXT_EXPORTS.items()} +def doc_target_not_found( + model, target_doc_name: str, target_doc_package: Optional[str] +) -> NoReturn: + raise DocTargetNotFound( + node=model, target_doc_name=target_doc_name, target_doc_package=target_doc_package + ) + + +def doc_invalid_args(model, args) -> NoReturn: + raise InvalidDocArgs(node=model, args=args) + + +def ref_bad_context(model, args) -> NoReturn: + raise RefBadContext(node=model, args=args) + + +def metric_invalid_args(model, args) -> NoReturn: + raise MetricInvalidArgs(node=model, args=args) + + +def ref_invalid_args(model, args) -> NoReturn: + raise RefInvalidArgs(node=model, args=args) + + +def invalid_bool_error(got_value, macro_name) -> NoReturn: + raise InvalidBoolean(return_value=got_value, macro_name=macro_name) + + +def invalid_type_error(method_name, arg_name, got_value, expected_type) -> NoReturn: + """Raise a CompilationException when an adapter method available to macros + has changed. + """ + raise InvalidMacroArgType(method_name, arg_name, got_value, expected_type) + + +def disallow_secret_env_var(env_var_name) -> NoReturn: + """Raise an error when a secret env var is referenced outside allowed + rendering contexts""" + raise DisallowSecretEnvVar(env_var_name) + + +def raise_parsing_error(msg, node=None) -> NoReturn: + raise ParsingException(msg, node) + + +# These are the exceptions functions that were not called within dbt-core but will remain here but deprecated to give a chance to rework +# TODO: is this valid? Should I create a special exception class for this? +def raise_unrecognized_credentials_type(typename, supported_types): + msg = 'Unrecognized credentials type "{}" - supported types are ({})'.format( + typename, ", ".join('"{}"'.format(t) for t in supported_types) + ) + raise CompilationException(msg) + + +def raise_patch_targets_not_found(patches): + patch_list = "\n\t".join( + f"model {p.name} (referenced in path {p.original_file_path})" for p in patches.values() + ) + msg = f"dbt could not find models for the following patches:\n\t{patch_list}" + raise CompilationException(msg) + + +def multiple_matching_relations(kwargs, matches): + raise RelationReturnedMultipleResults(kwargs, matches) + + +# while this isn't in our code I wouldn't be surpised it's in adapter code +def materialization_not_available(model, adapter_type): + raise MaterializationNotAvailable(model, adapter_type) + + +def macro_not_found(model, target_macro_id): + msg = f"'{model.unique_id}' references macro '{target_macro_id}' which is not defined!" + raise CompilationException(msg=msg, node=model) diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index 21bc74fbfc5..9c245214d83 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -18,7 +18,7 @@ from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ManifestNode, BaseNode from dbt.contracts.graph.unparsed import UnparsedNode, Docs -from dbt.exceptions import ParsingException, validator_error_message, InternalException +from dbt.exceptions import InternalException, InvalidConfigUpdate, InvalidDictParse from dbt import hooks from dbt.node_types import NodeType, ModelLanguage from dbt.parser.search import FileBlock @@ -216,7 +216,6 @@ def _create_parsetime_node( try: return self.parse_from_dict(dct, validate=True) except ValidationError as exc: - msg = validator_error_message(exc) # this is a bit silly, but build an UnparsedNode just for error # message reasons node = self._create_error_node( @@ -225,7 +224,7 @@ def _create_parsetime_node( original_file_path=block.path.original_file_path, raw_code=block.contents, ) - raise ParsingException(msg, node=node) + raise InvalidDictParse(exc, node=node) def _context_for(self, parsed_node: IntermediateNode, config: ContextConfig) -> Dict[str, Any]: return generate_parser_model_context(parsed_node, self.root_project, self.manifest, config) @@ -364,8 +363,7 @@ def render_update(self, node: IntermediateNode, config: ContextConfig) -> None: self.update_parsed_node_config(node, config, context=context) except ValidationError as exc: # we got a ValidationError - probably bad types in config() - msg = validator_error_message(exc) - raise ParsingException(msg, node=node) from exc + raise InvalidConfigUpdate(exc, node=node) from exc def add_result_node(self, block: FileBlock, node: ManifestNode): if node.config.enabled: diff --git a/core/dbt/parser/generic_test_builders.py b/core/dbt/parser/generic_test_builders.py index 3b1149e53a5..af0282c953f 100644 --- a/core/dbt/parser/generic_test_builders.py +++ b/core/dbt/parser/generic_test_builders.py @@ -21,7 +21,19 @@ UnparsedNodeUpdate, UnparsedExposure, ) -from dbt.exceptions import raise_compiler_error, raise_parsing_error, UndefinedMacroException +from dbt.exceptions import ( + CustomMacroPopulatingConfigValues, + SameKeyNested, + TagNotString, + TagsNotListOfStrings, + TestArgIncludesModel, + TestArgsNotDict, + TestDefinitionDictLength, + TestInvalidType, + TestNameNotString, + UnexpectedTestNamePattern, + UndefinedMacroException, +) from dbt.parser.search import FileBlock @@ -222,9 +234,7 @@ def __init__( test_name, test_args = self.extract_test_args(test, column_name) self.args: Dict[str, Any] = test_args if "model" in self.args: - raise_compiler_error( - 'Test arguments include "model", which is a reserved argument', - ) + raise TestArgIncludesModel() self.package_name: str = package_name self.target: Testable = target @@ -232,9 +242,7 @@ def __init__( match = self.TEST_NAME_PATTERN.match(test_name) if match is None: - raise_compiler_error( - "Test name string did not match expected pattern: {}".format(test_name) - ) + raise UnexpectedTestNamePattern(test_name) groups = match.groupdict() self.name: str = groups["test_name"] @@ -251,9 +259,7 @@ def __init__( value = self.args.pop(key, None) # 'modifier' config could be either top level arg or in config if value and "config" in self.args and key in self.args["config"]: - raise_compiler_error( - "Test cannot have the same key at the top-level and in config" - ) + raise SameKeyNested() if not value and "config" in self.args: value = self.args["config"].pop(key, None) if isinstance(value, str): @@ -261,22 +267,12 @@ def __init__( try: value = get_rendered(value, render_ctx, native=True) except UndefinedMacroException as e: - - # Generic tests do not include custom macros in the Jinja - # rendering context, so this will almost always fail. As it - # currently stands, the error message is inscrutable, which - # has caused issues for some projects migrating from - # pre-0.20.0 to post-0.20.0. - # See https://github.com/dbt-labs/dbt-core/issues/4103 - # and https://github.com/dbt-labs/dbt-core/issues/5294 - raise_compiler_error( - f"The {self.target.name}.{column_name} column's " - f'"{self.name}" test references an undefined ' - f"macro in its {key} configuration argument. " - f"The macro {e.msg}.\n" - "Please note that the generic test configuration parser " - "currently does not support using custom macros to " - "populate configuration values" + raise CustomMacroPopulatingConfigValues( + target_name=self.target.name, + column_name=column_name, + name=self.name, + key=key, + err_msg=e.msg ) if value is not None: @@ -314,9 +310,7 @@ def _bad_type(self) -> TypeError: @staticmethod def extract_test_args(test, name=None) -> Tuple[str, Dict[str, Any]]: if not isinstance(test, dict): - raise_parsing_error( - "test must be dict or str, got {} (value {})".format(type(test), test) - ) + raise TestInvalidType(test) # If the test is a dictionary with top-level keys, the test name is "test_name" # and the rest are arguments @@ -330,20 +324,13 @@ def extract_test_args(test, name=None) -> Tuple[str, Dict[str, Any]]: else: test = list(test.items()) if len(test) != 1: - raise_parsing_error( - "test definition dictionary must have exactly one key, got" - " {} instead ({} keys)".format(test, len(test)) - ) + raise TestDefinitionDictLength(test) test_name, test_args = test[0] if not isinstance(test_args, dict): - raise_parsing_error( - "test arguments must be dict, got {} (value {})".format(type(test_args), test_args) - ) + raise TestArgsNotDict(test_args) if not isinstance(test_name, str): - raise_parsing_error( - "test name must be a str, got {} (value {})".format(type(test_name), test_name) - ) + raise TestNameNotString(test_name) test_args = deepcopy(test_args) if name is not None: test_args["column_name"] = name @@ -434,12 +421,10 @@ def tags(self) -> List[str]: if isinstance(tags, str): tags = [tags] if not isinstance(tags, list): - raise_compiler_error( - f"got {tags} ({type(tags)}) for tags, expected a list of strings" - ) + raise TagsNotListOfStrings(tags) for tag in tags: if not isinstance(tag, str): - raise_compiler_error(f"got {tag} ({type(tag)}) for tag, expected a str") + raise TagNotString(tag) return tags[:] def macro_name(self) -> str: diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 2e284b43cfa..9da68736031 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -71,9 +71,7 @@ ResultNode, ) from dbt.contracts.util import Writable -from dbt.exceptions import ( - target_not_found, -) +from dbt.exceptions import TargetNotFound, AmbiguousAlias from dbt.parser.base import Parser from dbt.parser.analysis import AnalysisParser from dbt.parser.generic_test import GenericTestParser @@ -989,7 +987,7 @@ def invalid_target_fail_unless_test( ) ) else: - target_not_found( + raise TargetNotFound( node=node, target_name=target_name, target_kind=target_kind, @@ -1017,11 +1015,11 @@ def _check_resource_uniqueness( existing_node = names_resources.get(name) if existing_node is not None: - dbt.exceptions.raise_duplicate_resource_name(existing_node, node) + raise dbt.exceptions.DuplicateResourceName(existing_node, node) existing_alias = alias_resources.get(full_node_name) if existing_alias is not None: - dbt.exceptions.raise_ambiguous_alias(existing_alias, node, full_node_name) + raise AmbiguousAlias(node_1=existing_alias, node_2=node, duped_name=full_node_name) names_resources[name] = node alias_resources[full_node_name] = node diff --git a/core/dbt/parser/models.py b/core/dbt/parser/models.py index 8303e2f9c52..41ddfe0a5f3 100644 --- a/core/dbt/parser/models.py +++ b/core/dbt/parser/models.py @@ -29,7 +29,13 @@ # New for Python models :p import ast from dbt.dataclass_schema import ValidationError -from dbt.exceptions import ParsingException, validator_error_message, UndefinedMacroException +from dbt.exceptions import ( + InvalidModelConfig, + ParsingException, + PythonLiteralEval, + PythonParsingException, + UndefinedMacroException, +) dbt_function_key_words = set(["ref", "source", "config", "get"]) @@ -91,12 +97,7 @@ def _safe_eval(self, node): try: return ast.literal_eval(node) except (SyntaxError, ValueError, TypeError, MemoryError, RecursionError) as exc: - msg = validator_error_message( - f"Error when trying to literal_eval an arg to dbt.ref(), dbt.source(), dbt.config() or dbt.config.get() \n{exc}\n" - "https://docs.python.org/3/library/ast.html#ast.literal_eval\n" - "In dbt python model, `dbt.ref`, `dbt.source`, `dbt.config`, `dbt.config.get` function args only support Python literal structures" - ) - raise ParsingException(msg, node=self.dbt_node) from exc + raise PythonLiteralEval(exc, node=self.dbt_node) from exc def _get_call_literals(self, node): # List of literals @@ -199,8 +200,7 @@ def parse_python_model(self, node, config, context): try: tree = ast.parse(node.raw_code, filename=node.original_file_path) except SyntaxError as exc: - msg = validator_error_message(exc) - raise ParsingException(f"{msg}\n{exc.text}", node=node) from exc + raise PythonParsingException(exc, node=node) from exc # We are doing a validator and a parser because visit_FunctionDef in parser # would actually make the parser not doing the visit_Calls any more @@ -251,8 +251,7 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None: except ValidationError as exc: # we got a ValidationError - probably bad types in config() - msg = validator_error_message(exc) - raise ParsingException(msg, node=node) from exc + raise InvalidModelConfig(exc, node=node) from exc return elif not flags.STATIC_PARSER: diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 831647d0322..b5fd8558889 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -50,16 +50,22 @@ UnparsedSourceDefinition, ) from dbt.exceptions import ( - validator_error_message, + CompilationException, + DuplicateMacroPatchName, + DuplicatePatchPath, + DuplicateSourcePatchName, JSONValidationException, - raise_invalid_property_yml_version, - ValidationException, - ParsingException, - raise_duplicate_patch_name, - raise_duplicate_macro_patch_name, InternalException, - raise_duplicate_source_patch_name, - CompilationException, + InvalidSchemaConfig, + InvalidTestConfig, + ParsingException, + PropertyYMLInvalidTag, + PropertyYMLMissingVersion, + PropertyYMLVersionNotInt, + ValidationException, + YamlLoadFailure, + YamlParseDictFailure, + YamlParseListFailure, ) from dbt.events.functions import warn_or_error from dbt.events.types import WrongResourceSchemaFile, NoNodeForYamlKey, MacroPatchNotFound @@ -91,34 +97,13 @@ ) -def error_context( - path: str, - key: str, - data: Any, - cause: Union[str, ValidationException, JSONValidationException], -) -> str: - """Provide contextual information about an error while parsing""" - if isinstance(cause, str): - reason = cause - elif isinstance(cause, ValidationError): - reason = validator_error_message(cause) - else: - reason = cause.msg - return "Invalid {key} config given in {path} @ {key}: {data} - {reason}".format( - key=key, path=path, data=data, reason=reason - ) - - def yaml_from_file(source_file: SchemaSourceFile) -> Dict[str, Any]: """If loading the yaml fails, raise an exception.""" path = source_file.path.relative_path try: return load_yaml_text(source_file.contents, source_file.path) except ValidationException as e: - reason = validator_error_message(e) - raise ParsingException( - "Error reading {}: {} - {}".format(source_file.project_name, path, reason) - ) + raise YamlLoadFailure(source_file.project_name, path, e) class ParserRef: @@ -262,7 +247,6 @@ def get_hashable_md(data: Union[str, int, float, List, Dict]) -> Union[str, List GenericTestNode.validate(dct) return GenericTestNode.from_dict(dct) except ValidationError as exc: - msg = validator_error_message(exc) # this is a bit silly, but build an UnparsedNode just for error # message reasons node = self._create_error_node( @@ -271,7 +255,7 @@ def get_hashable_md(data: Union[str, int, float, List, Dict]) -> Union[str, List original_file_path=target.original_file_path, raw_code=raw_code, ) - raise ParsingException(msg, node=node) from exc + raise InvalidTestConfig(exc, node) # lots of time spent in this method def _parse_generic_test( @@ -413,8 +397,7 @@ def render_test_update(self, node, config, builder, schema_file_id): # env_vars should have been updated in the context env_var method except ValidationError as exc: # we got a ValidationError - probably bad types in config() - msg = validator_error_message(exc) - raise ParsingException(msg, node=node) from exc + raise InvalidSchemaConfig(exc, node=node) from exc def parse_node(self, block: GenericTestBlock) -> GenericTestNode: """In schema parsing, we rewrite most of the part of parse_node that @@ -554,25 +537,16 @@ def parse_file(self, block: FileBlock, dct: Dict = None) -> None: def check_format_version(file_path, yaml_dct) -> None: if "version" not in yaml_dct: - raise_invalid_property_yml_version( - file_path, - "the yml property file {} is missing a version tag".format(file_path), - ) + raise PropertyYMLMissingVersion(file_path) version = yaml_dct["version"] # if it's not an integer, the version is malformed, or not # set. Either way, only 'version: 2' is supported. if not isinstance(version, int): - raise_invalid_property_yml_version( - file_path, - "its 'version:' tag must be an integer (e.g. version: 2)." - " {} is not an integer".format(version), - ) + raise PropertyYMLVersionNotInt(file_path, version) + if version != 2: - raise_invalid_property_yml_version( - file_path, - "its 'version:' tag is set to {}. Only 2 is supported".format(version), - ) + raise PropertyYMLInvalidTag(file_path, version) Parsed = TypeVar("Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch) @@ -633,8 +607,9 @@ def get_key_dicts(self) -> Iterable[Dict[str, Any]]: # check that entry is a dict and that all dict values # are strings if coerce_dict_str(entry) is None: - msg = error_context(path, self.key, data, "expected a dict with string keys") - raise ParsingException(msg) + raise YamlParseListFailure( + path, self.key, data, "expected a dict with string keys" + ) if "name" not in entry: raise ParsingException("Entry did not contain a name") @@ -681,8 +656,7 @@ def _target_from_dict(self, cls: Type[T], data: Dict[str, Any]) -> T: cls.validate(data) return cls.from_dict(data) except (ValidationError, JSONValidationException) as exc: - msg = error_context(path, self.key, data, exc) - raise ParsingException(msg) from exc + raise YamlParseDictFailure(path, self.key, data, exc) # The other parse method returns TestBlocks. This one doesn't. # This takes the yaml dictionaries in 'sources' keys and uses them @@ -703,7 +677,7 @@ def parse(self) -> List[TestBlock]: # source patches must be unique key = (patch.overrides, patch.name) if key in self.manifest.source_patches: - raise_duplicate_source_patch_name(patch, self.manifest.source_patches[key]) + raise DuplicateSourcePatchName(patch, self.manifest.source_patches[key]) self.manifest.source_patches[key] = patch source_file.source_patches.append(key) else: @@ -807,8 +781,7 @@ def get_unparsed_target(self) -> Iterable[NonSourceTarget]: self.normalize_docs_attribute(data, path) node = self._target_type().from_dict(data) except (ValidationError, JSONValidationException) as exc: - msg = error_context(path, self.key, data, exc) - raise ParsingException(msg) from exc + raise YamlParseDictFailure(path, self.key, data, exc) else: yield node @@ -932,7 +905,7 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: if node: if node.patch_path: package_name, existing_file_path = node.patch_path.split("://") - raise_duplicate_patch_name(patch, existing_file_path) + raise DuplicatePatchPath(patch, existing_file_path) source_file.append_patch(patch.yaml_key, node.unique_id) # re-calculate the node config with the patch config. Always do this @@ -988,7 +961,7 @@ def parse_patch(self, block: TargetBlock[UnparsedMacroUpdate], refs: ParserRef) return if macro.patch_path: package_name, existing_file_path = macro.patch_path.split("://") - raise_duplicate_macro_patch_name(patch, existing_file_path) + raise DuplicateMacroPatchName(patch, existing_file_path) source_file.macro_patches[patch.name] = unique_id macro.patch(patch) @@ -1091,8 +1064,7 @@ def parse(self): UnparsedExposure.validate(data) unparsed = UnparsedExposure.from_dict(data) except (ValidationError, JSONValidationException) as exc: - msg = error_context(self.yaml.path, self.key, data, exc) - raise ParsingException(msg) from exc + raise YamlParseDictFailure(self.yaml.path, self.key, data, exc) self.parse_exposure(unparsed) @@ -1209,6 +1181,5 @@ def parse(self): unparsed = UnparsedMetric.from_dict(data) except (ValidationError, JSONValidationException) as exc: - msg = error_context(self.yaml.path, self.key, data, exc) - raise ParsingException(msg) from exc + raise YamlParseDictFailure(self.yaml.path, self.key, data, exc) self.parse_metric(unparsed) diff --git a/core/dbt/parser/snapshots.py b/core/dbt/parser/snapshots.py index 7fc46d1a05a..dffc7d90641 100644 --- a/core/dbt/parser/snapshots.py +++ b/core/dbt/parser/snapshots.py @@ -4,7 +4,7 @@ from dbt.dataclass_schema import ValidationError from dbt.contracts.graph.nodes import IntermediateSnapshotNode, SnapshotNode -from dbt.exceptions import ParsingException, validator_error_message +from dbt.exceptions import InvalidSnapshopConfig from dbt.node_types import NodeType from dbt.parser.base import SQLParser from dbt.parser.search import BlockContents, BlockSearcher, FileBlock @@ -68,7 +68,7 @@ def transform(self, node: IntermediateSnapshotNode) -> SnapshotNode: self.set_snapshot_attributes(parsed_node) return parsed_node except ValidationError as exc: - raise ParsingException(validator_error_message(exc), node) + raise InvalidSnapshopConfig(exc, node) def parse_file(self, file_block: FileBlock) -> None: blocks = BlockSearcher( diff --git a/core/dbt/task/generate.py b/core/dbt/task/generate.py index 48db2e772ba..87723a530a1 100644 --- a/core/dbt/task/generate.py +++ b/core/dbt/task/generate.py @@ -22,7 +22,7 @@ ColumnMetadata, CatalogArtifact, ) -from dbt.exceptions import InternalException +from dbt.exceptions import InternalException, AmbiguousCatalogMatch from dbt.include.global_project import DOCS_INDEX_FILE_PATH from dbt.events.functions import fire_event from dbt.events.types import ( @@ -119,7 +119,7 @@ def make_unique_id_map( unique_ids = source_map.get(table.key(), set()) for unique_id in unique_ids: if unique_id in sources: - dbt.exceptions.raise_ambiguous_catalog_match( + raise AmbiguousCatalogMatch( unique_id, sources[unique_id].to_dict(omit_none=True), table.to_dict(omit_none=True), diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index 5b88d039904..bc8f9a2de75 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -23,9 +23,9 @@ from dbt.exceptions import ( CompilationException, InternalException, + MissingMaterialization, RuntimeException, ValidationException, - missing_materialization, ) from dbt.events.functions import fire_event, get_invocation_id, info from dbt.events.types import ( @@ -252,7 +252,7 @@ def execute(self, model, manifest): ) if materialization_macro is None: - missing_materialization(model, self.adapter.type()) + raise MissingMaterialization(model=model, adapter_type=self.adapter.type()) if "config" not in context: raise InternalException( @@ -400,7 +400,7 @@ def safe_run_hooks( thread_id="main", timing=[], message=f"{hook_type.value} failed, error:\n {exc.msg}", - adapter_response=exc.msg, + adapter_response={}, execution_time=0, failures=1, ) diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 226005497e4..14005203296 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -243,7 +243,7 @@ def call_runner(self, runner): if result.status in (NodeStatus.Error, NodeStatus.Fail) and fail_fast: self._raise_next_tick = FailFastException( - message="Failing early due to test failure or runtime error", + msg="Failing early due to test failure or runtime error", result=result, node=getattr(result, "node", None), ) diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index e48dc94e4e4..26d6d46f028 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -21,7 +21,11 @@ LogTestResult, LogStartLine, ) -from dbt.exceptions import InternalException, invalid_bool_error, missing_materialization +from dbt.exceptions import ( + InternalException, + InvalidBoolean, + MissingMaterialization, +) from dbt.graph import ( ResourceTypeSelector, ) @@ -47,7 +51,7 @@ def convert_bool_type(field) -> bool: try: return bool(strtobool(field)) # type: ignore except ValueError: - raise invalid_bool_error(field, "get_test_sql") + raise InvalidBoolean(field, "get_test_sql") # need this so we catch both true bools and 0/1 return bool(field) @@ -97,7 +101,7 @@ def execute_test( ) if materialization_macro is None: - missing_materialization(test, self.adapter.type()) + raise MissingMaterialization(model=test, adapter_type=self.adapter.type()) if "config" not in context: raise InternalException( diff --git a/core/dbt/utils.py b/core/dbt/utils.py index b7cc6475319..987371b6b02 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -15,7 +15,7 @@ from pathlib import PosixPath, WindowsPath from contextlib import contextmanager -from dbt.exceptions import ConnectionException +from dbt.exceptions import ConnectionException, DuplicateAlias from dbt.events.functions import fire_event from dbt.events.types import RetryExternalCall, RecordRetryException from dbt import flags @@ -365,7 +365,7 @@ def translate_mapping(self, kwargs: Mapping[str, Any]) -> Dict[str, Any]: for key, value in kwargs.items(): canonical_key = self.aliases.get(key, key) if canonical_key in result: - dbt.exceptions.raise_duplicate_alias(kwargs, self.aliases, canonical_key) + raise DuplicateAlias(kwargs, self.aliases, canonical_key) result[canonical_key] = self.translate_value(value) return result diff --git a/plugins/postgres/dbt/adapters/postgres/impl.py b/plugins/postgres/dbt/adapters/postgres/impl.py index 3664e8d2a51..78b86234eae 100644 --- a/plugins/postgres/dbt/adapters/postgres/impl.py +++ b/plugins/postgres/dbt/adapters/postgres/impl.py @@ -8,7 +8,13 @@ from dbt.adapters.postgres import PostgresColumn from dbt.adapters.postgres import PostgresRelation from dbt.dataclass_schema import dbtClassMixin, ValidationError -import dbt.exceptions +from dbt.exceptions import ( + CrossDbReferenceProhibited, + IndexConfigNotDict, + InvalidIndexConfig, + RuntimeException, + UnexpectedDbReference, +) import dbt.utils @@ -40,14 +46,9 @@ def parse(cls, raw_index) -> Optional["PostgresIndexConfig"]: cls.validate(raw_index) return cls.from_dict(raw_index) except ValidationError as exc: - msg = dbt.exceptions.validator_error_message(exc) - dbt.exceptions.raise_compiler_error(f"Could not parse index config: {msg}") + raise InvalidIndexConfig(exc) except TypeError: - dbt.exceptions.raise_compiler_error( - f"Invalid index config:\n" - f" Got: {raw_index}\n" - f' Expected a dictionary with at minimum a "columns" key' - ) + raise IndexConfigNotDict(raw_index) @dataclass @@ -73,11 +74,7 @@ def verify_database(self, database): database = database.strip('"') expected = self.config.credentials.database if database.lower() != expected.lower(): - raise dbt.exceptions.NotImplementedException( - "Cross-db references not allowed in {} ({} vs {})".format( - self.type(), database, expected - ) - ) + raise UnexpectedDbReference(self.type(), database, expected) # return an empty string on success so macros can call this return "" @@ -110,12 +107,8 @@ def _get_catalog_schemas(self, manifest): schemas = super()._get_catalog_schemas(manifest) try: return schemas.flatten() - except dbt.exceptions.RuntimeException as exc: - dbt.exceptions.raise_compiler_error( - "Cross-db references not allowed in adapter {}: Got {}".format( - self.type(), exc.msg - ) - ) + except RuntimeException as exc: + raise CrossDbReferenceProhibited(self.type(), exc.msg) def _link_cached_relations(self, manifest): schemas: Set[str] = set() diff --git a/tests/functional/duplicates/test_duplicate_model.py b/tests/functional/duplicates/test_duplicate_model.py index 031ba6236c0..fbcd1b79671 100644 --- a/tests/functional/duplicates/test_duplicate_model.py +++ b/tests/functional/duplicates/test_duplicate_model.py @@ -1,6 +1,6 @@ import pytest -from dbt.exceptions import CompilationException +from dbt.exceptions import CompilationException, DuplicateResourceName from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import run_dbt, get_manifest @@ -108,7 +108,7 @@ def packages(self): def test_duplicate_model_enabled_across_packages(self, project): run_dbt(["deps"]) message = "dbt found two models with the name" - with pytest.raises(CompilationException) as exc: + with pytest.raises(DuplicateResourceName) as exc: run_dbt(["run"]) assert message in str(exc.value) diff --git a/tests/functional/exit_codes/test_exit_codes.py b/tests/functional/exit_codes/test_exit_codes.py index 955953a0dc0..54b5cb6865e 100644 --- a/tests/functional/exit_codes/test_exit_codes.py +++ b/tests/functional/exit_codes/test_exit_codes.py @@ -99,7 +99,7 @@ def packages(self): } def test_deps_fail(self, project): - with pytest.raises(dbt.exceptions.InternalException) as exc: + with pytest.raises(dbt.exceptions.GitCheckoutError) as exc: run_dbt(['deps']) expected_msg = "Error checking out spec='bad-branch'" assert expected_msg in str(exc.value) diff --git a/tests/functional/schema_tests/test_schema_v2_tests.py b/tests/functional/schema_tests/test_schema_v2_tests.py index 00c14cd711b..44a6696931b 100644 --- a/tests/functional/schema_tests/test_schema_v2_tests.py +++ b/tests/functional/schema_tests/test_schema_v2_tests.py @@ -95,7 +95,7 @@ alt_local_utils__macros__type_timestamp_sql, all_quotes_schema__schema_yml, ) -from dbt.exceptions import ParsingException, CompilationException +from dbt.exceptions import ParsingException, CompilationException, DuplicateResourceName from dbt.contracts.results import TestStatus @@ -904,9 +904,9 @@ def test_generic_test_collision( project, ): """These tests collide, since only the configs differ""" - with pytest.raises(CompilationException) as exc: + with pytest.raises(DuplicateResourceName) as exc: run_dbt() - assert "dbt found two tests with the name" in str(exc) + assert "dbt found two tests with the name" in str(exc.value) class TestGenericTestsConfigCustomMacros: