diff --git a/.changes/unreleased/Features-20230914-074429.yaml b/.changes/unreleased/Features-20230914-074429.yaml new file mode 100644 index 00000000000..980f932b99c --- /dev/null +++ b/.changes/unreleased/Features-20230914-074429.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Add support for optional label in semantic_models, measures, dimensions and + entities. +time: 2023-09-14T07:44:29.828199-05:00 +custom: + Author: emmyoop + Issue: "8595" diff --git a/.changes/unreleased/Features-20230915-123733.yaml b/.changes/unreleased/Features-20230915-123733.yaml new file mode 100644 index 00000000000..146ad8ef89a --- /dev/null +++ b/.changes/unreleased/Features-20230915-123733.yaml @@ -0,0 +1,6 @@ +kind: Features +body: 'Allow adapters to include package logs in dbt standard logging ' +time: 2023-09-15T12:37:33.862862-07:00 +custom: + Author: colin-rogers-dbt + Issue: "7859" diff --git a/.changes/unreleased/Features-20230918-150855.yaml b/.changes/unreleased/Features-20230918-150855.yaml new file mode 100644 index 00000000000..b54b3447b7a --- /dev/null +++ b/.changes/unreleased/Features-20230918-150855.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support quoted parameter list for MultiOption CLI options. +time: 2023-09-18T15:08:55.625412-05:00 +custom: + Author: emmyoop + Issue: "8598" diff --git a/.changes/unreleased/Features-20230922-112531.yaml b/.changes/unreleased/Features-20230922-112531.yaml new file mode 100644 index 00000000000..99af15448f5 --- /dev/null +++ b/.changes/unreleased/Features-20230922-112531.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Adding `date_spine` macro (and supporting macros) from dbt-utils to dbt-core +time: 2023-09-22T11:25:31.383445-07:00 +custom: + Author: QMalcolm + Issue: "8172" diff --git a/.changes/unreleased/Features-20230922-150754.yaml b/.changes/unreleased/Features-20230922-150754.yaml new file mode 100644 index 00000000000..6492c3d934a --- /dev/null +++ b/.changes/unreleased/Features-20230922-150754.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support `fill_nulls_with` and `join_to_timespine` for metric nodes +time: 2023-09-22T15:07:54.981752-07:00 +custom: + Author: QMalcolm + Issue: "8593" diff --git a/.changes/unreleased/Fixes-20230920-165635.yaml b/.changes/unreleased/Fixes-20230920-165635.yaml new file mode 100644 index 00000000000..d5c36319ee4 --- /dev/null +++ b/.changes/unreleased/Fixes-20230920-165635.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Lower bound of `8.0.2` for `click` +time: 2023-09-20T16:56:35.484024-06:00 +custom: + Author: dylan-murray dbeatty10 + Issue: "8683" diff --git a/.changes/unreleased/Under the Hood-20230831-164435.yaml b/.changes/unreleased/Under the Hood-20230831-164435.yaml new file mode 100644 index 00000000000..efa8a42cece --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230831-164435.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Added more type annotations. +time: 2023-08-31T16:44:35.737954-04:00 +custom: + Author: peterallenwebb + Issue: "8537" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f3eb6fbf40f..2831dd14422 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -30,3 +30,4 @@ resolves # - [ ] I have run this code in development and it appears to resolve the stated issue - [ ] This PR includes tests, or tests are not required/relevant for this PR - [ ] This PR has no interface changes (e.g. macros, cli, logs, json artifacts, config files, adapter interface, etc) or this PR has already received feedback and approval from Product or DX +- [ ] This PR includes [type annotations](https://docs.python.org/3/library/typing.html) for new and modified functions diff --git a/core/dbt/adapters/base/connections.py b/core/dbt/adapters/base/connections.py index 23a8b6e6588..1731924dffb 100644 --- a/core/dbt/adapters/base/connections.py +++ b/core/dbt/adapters/base/connections.py @@ -72,7 +72,7 @@ class BaseConnectionManager(metaclass=abc.ABCMeta): TYPE: str = NotImplemented - def __init__(self, profile: AdapterRequiredConfig): + def __init__(self, profile: AdapterRequiredConfig) -> None: self.profile = profile self.thread_connections: Dict[Hashable, Connection] = {} self.lock: RLock = flags.MP_CONTEXT.RLock() diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index dd147dce845..37ab45f8292 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -222,7 +222,7 @@ class BaseAdapter(metaclass=AdapterMeta): ConstraintType.foreign_key: ConstraintSupport.ENFORCED, } - def __init__(self, config): + def __init__(self, config) -> None: self.config = config self.cache = RelationsCache() self.connections = self.ConnectionManager(config) diff --git a/core/dbt/adapters/base/plugin.py b/core/dbt/adapters/base/plugin.py index 58481f75439..5faa2163a4a 100644 --- a/core/dbt/adapters/base/plugin.py +++ b/core/dbt/adapters/base/plugin.py @@ -29,7 +29,7 @@ def __init__( credentials: Type[Credentials], include_path: str, dependencies: Optional[List[str]] = None, - ): + ) -> None: self.adapter: Type[AdapterProtocol] = adapter self.credentials: Type[Credentials] = credentials diff --git a/core/dbt/adapters/base/query_headers.py b/core/dbt/adapters/base/query_headers.py index 19a21c7a02b..f1fde8dc027 100644 --- a/core/dbt/adapters/base/query_headers.py +++ b/core/dbt/adapters/base/query_headers.py @@ -11,7 +11,7 @@ class NodeWrapper: - def __init__(self, node): + def __init__(self, node) -> None: self._inner_node = node def __getattr__(self, name): @@ -57,7 +57,7 @@ def set(self, comment: Optional[str], append: bool): class MacroQueryStringSetter: - def __init__(self, config: AdapterRequiredConfig, manifest: Manifest): + def __init__(self, config: AdapterRequiredConfig, manifest: Manifest) -> None: self.manifest = manifest self.config = config diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 3e783de21e9..c5559729678 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -38,8 +38,8 @@ class _CachedRelation: :attr BaseRelation inner: The underlying dbt relation. """ - def __init__(self, inner): - self.referenced_by = {} + def __init__(self, inner) -> None: + self.referenced_by: Dict[_ReferenceKey, _CachedRelation] = {} self.inner = inner def __str__(self) -> str: diff --git a/core/dbt/adapters/protocol.py b/core/dbt/adapters/protocol.py index 13b9bd79968..1c58cc78bab 100644 --- a/core/dbt/adapters/protocol.py +++ b/core/dbt/adapters/protocol.py @@ -90,7 +90,7 @@ class AdapterProtocol( # type: ignore[misc] ConnectionManager: Type[ConnectionManager_T] connections: ConnectionManager_T - def __init__(self, config: AdapterRequiredConfig): + def __init__(self, config: AdapterRequiredConfig) -> None: ... @classmethod diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 863db6ed0e4..2678d53b6dd 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -57,7 +57,7 @@ def args_to_context(args: List[str]) -> Context: from dbt.cli.main import cli cli_ctx = cli.make_context(cli.name, args) - # Split args if they're a comma seperated string. + # Split args if they're a comma separated string. if len(args) == 1 and "," in args[0]: args = args[0].split(",") sub_command_name, sub_command, args = cli.resolve_command(cli_ctx, args) @@ -340,6 +340,10 @@ def add_fn(x): spinal_cased = k.replace("_", "-") + # MultiOption flags come back as lists, but we want to pass them as space separated strings + if isinstance(v, list): + v = " ".join(v) + if k == "macro" and command == CliCommand.RUN_OPERATION: add_fn(v) # None is a Singleton, False is a Flyweight, only one instance of each. diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index ab501e015f4..a8613693285 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -65,7 +65,7 @@ def __init__( self, manifest: Optional[Manifest] = None, callbacks: Optional[List[Callable[[EventMsg], None]]] = None, - ): + ) -> None: self.manifest = manifest if callbacks is None: diff --git a/core/dbt/cli/options.py b/core/dbt/cli/options.py index 3a42dddda80..bf0749ae002 100644 --- a/core/dbt/cli/options.py +++ b/core/dbt/cli/options.py @@ -2,19 +2,21 @@ import inspect import typing as t from click import Context +from click.parser import OptionParser, ParsingState from dbt.cli.option_types import ChoiceTuple # Implementation from: https://stackoverflow.com/a/48394004 # Note MultiOption options must be specified with type=tuple or type=ChoiceTuple (https://github.com/pallets/click/issues/2012) class MultiOption(click.Option): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: self.save_other_options = kwargs.pop("save_other_options", True) nargs = kwargs.pop("nargs", -1) assert nargs == -1, "nargs, if set, must be -1 not {}".format(nargs) super(MultiOption, self).__init__(*args, **kwargs) - self._previous_parser_process = None - self._eat_all_parser = None + # this makes mypy happy, setting these to None causes mypy failures + self._previous_parser_process = lambda *args, **kwargs: None + self._eat_all_parser = lambda *args, **kwargs: None # validate that multiple=True multiple = kwargs.pop("multiple", None) @@ -29,34 +31,35 @@ def __init__(self, *args, **kwargs): else: assert isinstance(option_type, ChoiceTuple), msg - def add_to_parser(self, parser, ctx): - def parser_process(value, state): + def add_to_parser(self, parser: OptionParser, ctx: Context): + def parser_process(value: str, state: ParsingState): # method to hook to the parser.process done = False - value = [value] + value_list = str.split(value, " ") if self.save_other_options: # grab everything up to the next option while state.rargs and not done: - for prefix in self._eat_all_parser.prefixes: + for prefix in self._eat_all_parser.prefixes: # type: ignore[attr-defined] if state.rargs[0].startswith(prefix): done = True if not done: - value.append(state.rargs.pop(0)) + value_list.append(state.rargs.pop(0)) else: # grab everything remaining - value += state.rargs + value_list += state.rargs state.rargs[:] = [] - value = tuple(value) + value_tuple = tuple(value_list) # call the actual process - self._previous_parser_process(value, state) + self._previous_parser_process(value_tuple, state) retval = super(MultiOption, self).add_to_parser(parser, ctx) for name in self.opts: our_parser = parser._long_opt.get(name) or parser._short_opt.get(name) if our_parser: - self._eat_all_parser = our_parser + self._eat_all_parser = our_parser # type: ignore[assignment] self._previous_parser_process = our_parser.process - our_parser.process = parser_process + # mypy doesnt like assingment to a method see https://github.com/python/mypy/issues/708 + our_parser.process = parser_process # type: ignore[method-assign] break return retval diff --git a/core/dbt/clients/agate_helper.py b/core/dbt/clients/agate_helper.py index 29db25fe570..a60446d2aed 100644 --- a/core/dbt/clients/agate_helper.py +++ b/core/dbt/clients/agate_helper.py @@ -165,7 +165,7 @@ class _NullMarker: class ColumnTypeBuilder(Dict[str, NullableAgateType]): - def __init__(self): + def __init__(self) -> None: super().__init__() def __setitem__(self, key, value): diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index f37e248515a..f42141d700a 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -125,7 +125,7 @@ def _get_tests_for_node(manifest: Manifest, unique_id: UniqueID) -> List[UniqueI class Linker: - def __init__(self, data=None): + def __init__(self, data=None) -> None: if data is None: data = {} self.graph = nx.DiGraph(**data) @@ -274,7 +274,7 @@ def get_graph_summary(self, manifest: Manifest) -> Dict[int, Dict[str, Any]]: class Compiler: - def __init__(self, config): + def __init__(self, config) -> None: self.config = config def initialize(self): diff --git a/core/dbt/config/profile.py b/core/dbt/config/profile.py index d31342223c2..acd06e6a8a7 100644 --- a/core/dbt/config/profile.py +++ b/core/dbt/config/profile.py @@ -83,7 +83,7 @@ def __init__( user_config: UserConfig, threads: int, credentials: Credentials, - ): + ) -> None: """Explicitly defining `__init__` to work around bug in Python 3.9.7 https://bugs.python.org/issue45081 """ diff --git a/core/dbt/config/renderer.py b/core/dbt/config/renderer.py index aba4791c86c..f29433454da 100644 --- a/core/dbt/config/renderer.py +++ b/core/dbt/config/renderer.py @@ -74,7 +74,7 @@ def _list_if_none_or_string(value): class ProjectPostprocessor(Dict[Keypath, Callable[[Any], Any]]): - def __init__(self): + def __init__(self) -> None: super().__init__() self[("on-run-start",)] = _list_if_none_or_string diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index 192653c9fd4..4d40628509f 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -407,7 +407,7 @@ def _get_project_directories(self) -> Iterator[Path]: class UnsetCredentials(Credentials): - def __init__(self): + def __init__(self) -> None: super().__init__("", "") @property diff --git a/core/dbt/contracts/connection.py b/core/dbt/contracts/connection.py index 9f2447f1fce..692f40f71b7 100644 --- a/core/dbt/contracts/connection.py +++ b/core/dbt/contracts/connection.py @@ -104,7 +104,7 @@ class LazyHandle: connection, updating the handle on the Connection. """ - def __init__(self, opener: Callable[[Connection], Connection]): + def __init__(self, opener: Callable[[Connection], Connection]) -> None: self.opener = opener def resolve(self, connection: Connection) -> Connection: diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index d4bdcbddcc1..9a1223a9ead 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -88,7 +88,7 @@ def find_unique_id_for_package(storage, key, package: Optional[PackageName]): class DocLookup(dbtClassMixin): - def __init__(self, manifest: "Manifest"): + def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) @@ -119,7 +119,7 @@ def perform_lookup(self, unique_id: UniqueID, manifest) -> Documentation: class SourceLookup(dbtClassMixin): - def __init__(self, manifest: "Manifest"): + def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) @@ -156,7 +156,7 @@ class RefableLookup(dbtClassMixin): _lookup_types: ClassVar[set] = set(NodeType.refable()) _versioned_types: ClassVar[set] = set(NodeType.versioned()) - def __init__(self, manifest: "Manifest"): + def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) @@ -267,7 +267,7 @@ def _find_unique_ids_for_package(self, key, package: Optional[PackageName]) -> L class MetricLookup(dbtClassMixin): - def __init__(self, manifest: "Manifest"): + def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, UniqueID]] = {} self.populate(manifest) @@ -306,7 +306,7 @@ class SemanticModelByMeasureLookup(dbtClassMixin): the semantic models in a manifest. """ - def __init__(self, manifest: "Manifest"): + def __init__(self, manifest: "Manifest") -> None: self.storage: DefaultDict[str, Dict[PackageName, UniqueID]] = defaultdict(dict) self.populate(manifest) @@ -355,7 +355,7 @@ def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SemanticM # This handles both models/seeds/snapshots and sources/metrics/exposures/semantic_models class DisabledLookup(dbtClassMixin): - def __init__(self, manifest: "Manifest"): + def __init__(self, manifest: "Manifest") -> None: self.storage: Dict[str, Dict[PackageName, List[Any]]] = {} self.populate(manifest) @@ -1427,12 +1427,12 @@ def __reduce_ex__(self, protocol): class MacroManifest(MacroMethods): - def __init__(self, macros): + def __init__(self, macros) -> None: self.macros = macros self.metadata = ManifestMetadata() # This is returned by the 'graph' context property # in the ProviderContext class. - self.flat_graph = {} + self.flat_graph: Dict[str, Any] = {} AnyManifest = Union[Manifest, MacroManifest] diff --git a/core/dbt/contracts/graph/metrics.py b/core/dbt/contracts/graph/metrics.py index c95604cb976..c5ce2fbd41b 100644 --- a/core/dbt/contracts/graph/metrics.py +++ b/core/dbt/contracts/graph/metrics.py @@ -9,7 +9,7 @@ class MetricReference(object): - def __init__(self, metric_name, package_name=None): + def __init__(self, metric_name, package_name=None) -> None: self.metric_name = metric_name self.package_name = package_name diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 35d2ee4d810..3aad00901eb 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1411,6 +1411,8 @@ class MetricInputMeasure(dbtClassMixin): name: str filter: Optional[WhereFilter] = None alias: Optional[str] = None + join_to_timespine: bool = False + fill_nulls_with: Optional[int] = None def measure_reference(self) -> MeasureReference: return MeasureReference(element_name=self.name) @@ -1571,6 +1573,7 @@ class SemanticModel(GraphNode): model: str node_relation: Optional[NodeRelation] description: Optional[str] = None + label: Optional[str] = None defaults: Optional[Defaults] = None entities: Sequence[Entity] = field(default_factory=list) measures: Sequence[Measure] = field(default_factory=list) diff --git a/core/dbt/contracts/graph/semantic_manifest.py b/core/dbt/contracts/graph/semantic_manifest.py index 8c599b1a7fa..23b9b5a6853 100644 --- a/core/dbt/contracts/graph/semantic_manifest.py +++ b/core/dbt/contracts/graph/semantic_manifest.py @@ -20,7 +20,7 @@ class SemanticManifest: - def __init__(self, manifest): + def __init__(self, manifest) -> None: self.manifest = manifest def validate(self) -> bool: diff --git a/core/dbt/contracts/graph/semantic_models.py b/core/dbt/contracts/graph/semantic_models.py index 2bb75382c46..bdcf1effbce 100644 --- a/core/dbt/contracts/graph/semantic_models.py +++ b/core/dbt/contracts/graph/semantic_models.py @@ -66,6 +66,7 @@ class Dimension(dbtClassMixin): name: str type: DimensionType description: Optional[str] = None + label: Optional[str] = None is_partition: bool = False type_params: Optional[DimensionTypeParams] = None expr: Optional[str] = None @@ -100,6 +101,7 @@ class Entity(dbtClassMixin): name: str type: EntityType description: Optional[str] = None + label: Optional[str] = None role: Optional[str] = None expr: Optional[str] = None @@ -136,6 +138,7 @@ class Measure(dbtClassMixin): name: str agg: AggregationType description: Optional[str] = None + label: Optional[str] = None create_metric: bool = False expr: Optional[str] = None agg_params: Optional[MeasureAggregationParameters] = None diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index dbe0bed9812..d06b13faa03 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -584,6 +584,8 @@ class UnparsedMetricInputMeasure(dbtClassMixin): name: str filter: Optional[str] = None alias: Optional[str] = None + join_to_timespine: bool = False + fill_nulls_with: Optional[int] = None @dataclass @@ -663,6 +665,7 @@ class UnparsedEntity(dbtClassMixin): name: str type: str # EntityType enum description: Optional[str] = None + label: Optional[str] = None role: Optional[str] = None expr: Optional[str] = None @@ -679,6 +682,7 @@ class UnparsedMeasure(dbtClassMixin): name: str agg: str # actually an enum description: Optional[str] = None + label: Optional[str] = None expr: Optional[Union[str, bool, int]] = None agg_params: Optional[MeasureAggregationParameters] = None non_additive_dimension: Optional[UnparsedNonAdditiveDimension] = None @@ -697,6 +701,7 @@ class UnparsedDimension(dbtClassMixin): name: str type: str # actually an enum description: Optional[str] = None + label: Optional[str] = None is_partition: bool = False type_params: Optional[UnparsedDimensionTypeParams] = None expr: Optional[str] = None @@ -708,6 +713,7 @@ class UnparsedSemanticModel(dbtClassMixin): model: str # looks like "ref(...)" config: Dict[str, Any] = field(default_factory=dict) description: Optional[str] = None + label: Optional[str] = None defaults: Optional[Defaults] = None entities: List[UnparsedEntity] = field(default_factory=list) measures: List[UnparsedMeasure] = field(default_factory=list) diff --git a/core/dbt/contracts/results.py b/core/dbt/contracts/results.py index 4d8a945a674..2278e5561a6 100644 --- a/core/dbt/contracts/results.py +++ b/core/dbt/contracts/results.py @@ -59,7 +59,7 @@ def to_msg_dict(self): # This is a context manager class collect_timing_info: - def __init__(self, name: str, callback: Callable[[TimingInfo], None]): + def __init__(self, name: str, callback: Callable[[TimingInfo], None]) -> None: self.timing_info = TimingInfo(name=name) self.callback = callback diff --git a/core/dbt/contracts/state.py b/core/dbt/contracts/state.py index 2bb025ea2c1..6627505e5a9 100644 --- a/core/dbt/contracts/state.py +++ b/core/dbt/contracts/state.py @@ -10,7 +10,7 @@ class PreviousState: - def __init__(self, state_path: Path, target_path: Path, project_root: Path): + def __init__(self, state_path: Path, target_path: Path, project_root: Path) -> None: self.state_path: Path = state_path self.target_path: Path = target_path self.project_root: Path = project_root diff --git a/core/dbt/events/adapter_endpoint.py b/core/dbt/events/adapter_endpoint.py index 7e5cf0cd1d3..938af608b72 100644 --- a/core/dbt/events/adapter_endpoint.py +++ b/core/dbt/events/adapter_endpoint.py @@ -1,7 +1,8 @@ import traceback from dataclasses import dataclass -from dbt.events.functions import fire_event +from dbt.events.functions import fire_event, EVENT_MANAGER from dbt.events.contextvars import get_node_info +from dbt.events.event_handler import set_package_logging from dbt.events.types import ( AdapterEventDebug, AdapterEventInfo, @@ -15,32 +16,32 @@ class AdapterLogger: name: str - def debug(self, msg, *args): + def debug(self, msg, *args) -> None: event = AdapterEventDebug( name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info() ) fire_event(event) - def info(self, msg, *args): + def info(self, msg, *args) -> None: event = AdapterEventInfo( name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info() ) fire_event(event) - def warning(self, msg, *args): + def warning(self, msg, *args) -> None: event = AdapterEventWarning( name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info() ) fire_event(event) - def error(self, msg, *args): + def error(self, msg, *args) -> None: event = AdapterEventError( name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info() ) fire_event(event) # The default exc_info=True is what makes this method different - def exception(self, msg, *args): + def exception(self, msg, *args) -> None: exc_info = str(traceback.format_exc()) event = AdapterEventError( name=self.name, @@ -51,8 +52,15 @@ def exception(self, msg, *args): ) fire_event(event) - def critical(self, msg, *args): + def critical(self, msg, *args) -> None: event = AdapterEventError( name=self.name, base_msg=str(msg), args=list(args), node_info=get_node_info() ) fire_event(event) + + @staticmethod + def set_adapter_dependency_log_level(package_name, level): + """By default, dbt suppresses non-dbt package logs. This method allows + you to set the log level for a specific package. + """ + set_package_logging(package_name, level, EVENT_MANAGER) diff --git a/core/dbt/events/base_types.py b/core/dbt/events/base_types.py index 711a0bfad72..f939abf3cc1 100644 --- a/core/dbt/events/base_types.py +++ b/core/dbt/events/base_types.py @@ -37,7 +37,7 @@ def get_pid() -> int: return os.getpid() -# in theory threads can change so we don't cache them. +# in theory threads can change, so we don't cache them. def get_thread_name() -> str: return threading.current_thread().name @@ -55,7 +55,7 @@ class EventLevel(str, Enum): class BaseEvent: """BaseEvent for proto message generated python events""" - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: class_name = type(self).__name__ msg_cls = getattr(types_pb2, class_name) if class_name == "Formatting" and len(args) > 0: @@ -100,9 +100,9 @@ def to_dict(self): self.pb_msg, preserving_proto_field_name=True, including_default_value_fields=True ) - def to_json(self): + def to_json(self) -> str: return MessageToJson( - self.pb_msg, preserving_proto_field_name=True, including_default_valud_fields=True + self.pb_msg, preserving_proto_field_name=True, including_default_value_fields=True ) def level_tag(self) -> EventLevel: diff --git a/core/dbt/events/event_handler.py b/core/dbt/events/event_handler.py new file mode 100644 index 00000000000..4a1731a6f58 --- /dev/null +++ b/core/dbt/events/event_handler.py @@ -0,0 +1,40 @@ +import logging +from typing import Union + +from dbt.events.base_types import EventLevel +from dbt.events.types import Note + +from dbt.events.eventmgr import IEventManager + +_log_level_to_event_level_map = { + logging.DEBUG: EventLevel.DEBUG, + logging.INFO: EventLevel.INFO, + logging.WARN: EventLevel.WARN, + logging.WARNING: EventLevel.WARN, + logging.ERROR: EventLevel.ERROR, + logging.CRITICAL: EventLevel.ERROR, +} + + +class DbtEventLoggingHandler(logging.Handler): + """A logging handler that wraps the EventManager + This allows non-dbt packages to log to the dbt event stream. + All logs are generated as "Note" events. + """ + + def __init__(self, event_manager: IEventManager, level): + super().__init__(level) + self.event_manager = event_manager + + def emit(self, record: logging.LogRecord): + note = Note(msg=record.getMessage()) + level = _log_level_to_event_level_map[record.levelno] + self.event_manager.fire_event(e=note, level=level) + + +def set_package_logging(package_name: str, log_level: Union[str, int], event_mgr: IEventManager): + """Attach dbt's custom logging handler to the package's logger.""" + log = logging.getLogger(package_name) + log.setLevel(log_level) + event_handler = DbtEventLoggingHandler(event_manager=event_mgr, level=log_level) + log.addHandler(event_handler) diff --git a/core/dbt/events/eventmgr.py b/core/dbt/events/eventmgr.py index 5c05047d69c..47fd58704fe 100644 --- a/core/dbt/events/eventmgr.py +++ b/core/dbt/events/eventmgr.py @@ -1,6 +1,6 @@ import os import traceback -from typing import Callable, List, Optional, Protocol +from typing import Callable, List, Optional, Protocol, Tuple from uuid import uuid4 from dbt.events.base_types import BaseEvent, EventLevel, msg_from_base_event, EventMsg @@ -38,7 +38,7 @@ def add_logger(self, config: LoggerConfig) -> None: ) self.loggers.append(logger) - def flush(self): + def flush(self) -> None: for logger in self.loggers: logger.flush() @@ -46,6 +46,7 @@ def flush(self): class IEventManager(Protocol): callbacks: List[Callable[[EventMsg], None]] invocation_id: str + loggers: List[_Logger] def fire_event(self, e: BaseEvent, level: Optional[EventLevel] = None) -> None: ... @@ -55,8 +56,9 @@ def add_logger(self, config: LoggerConfig) -> None: class TestEventManager(IEventManager): - def __init__(self): - self.event_history = [] + def __init__(self) -> None: + self.event_history: List[Tuple[BaseEvent, Optional[EventLevel]]] = [] + self.loggers = [] def fire_event(self, e: BaseEvent, level: Optional[EventLevel] = None) -> None: self.event_history.append((e, level)) diff --git a/core/dbt/events/format.py b/core/dbt/events/format.py index dad6cf9f355..4b173b18cca 100644 --- a/core/dbt/events/format.py +++ b/core/dbt/events/format.py @@ -44,13 +44,13 @@ def _pluralize(string: Union[str, NodeType]) -> str: return convert.pluralize() -def pluralize(count, string: Union[str, NodeType]): +def pluralize(count, string: Union[str, NodeType]) -> str: pluralized: str = str(string) if count != 1: pluralized = _pluralize(string) return f"{count} {pluralized}" -def timestamp_to_datetime_string(ts): +def timestamp_to_datetime_string(ts) -> str: timestamp_dt = datetime.fromtimestamp(ts.seconds + ts.nanos / 1e9) return timestamp_dt.strftime("%H:%M:%S.%f") diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 10b3c590846..bfb1fda5a39 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -95,7 +95,6 @@ def _get_stdout_config( level: EventLevel, log_cache_events: bool, ) -> LoggerConfig: - return LoggerConfig( name="stdout_log", level=level, @@ -127,6 +126,7 @@ def _get_logfile_config( level: EventLevel, log_file_max_bytes: int, ) -> LoggerConfig: + return LoggerConfig( name="file_log", line_format=line_format, @@ -170,7 +170,7 @@ def env_scrubber(msg: str) -> str: return scrub_secrets(msg, env_secrets()) -def cleanup_event_logger(): +def cleanup_event_logger() -> None: # Reset to a no-op manager to release streams associated with logs. This is # especially important for tests, since pytest replaces the stdout stream # during test runs, and closes the stream after the test is over. @@ -195,12 +195,12 @@ def cleanup_event_logger(): # used for integration tests -def capture_stdout_logs(stream: TextIO): +def capture_stdout_logs(stream: TextIO) -> None: global _CAPTURE_STREAM _CAPTURE_STREAM = stream -def stop_capture_stdout_logs(): +def stop_capture_stdout_logs() -> None: global _CAPTURE_STREAM _CAPTURE_STREAM = None @@ -234,7 +234,7 @@ def msg_to_dict(msg: EventMsg) -> dict: return msg_dict -def warn_or_error(event, node=None): +def warn_or_error(event, node=None) -> None: flags = get_flags() if flags.WARN_ERROR or flags.WARN_ERROR_OPTIONS.includes(type(event).__name__): @@ -296,6 +296,6 @@ def set_invocation_id() -> None: EVENT_MANAGER.invocation_id = str(uuid.uuid4()) -def ctx_set_event_manager(event_manager: IEventManager): +def ctx_set_event_manager(event_manager: IEventManager) -> None: global EVENT_MANAGER EVENT_MANAGER = event_manager diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index c51481d1fd9..009e441b301 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -52,31 +52,31 @@ def format_adapter_message(name, base_msg, args) -> str: class MainReportVersion(InfoLevel): - def code(self): + def code(self) -> str: return "A001" - def message(self): + def message(self) -> str: return f"Running with dbt{self.version}" class MainReportArgs(DebugLevel): - def code(self): + def code(self) -> str: return "A002" - def message(self): + def message(self) -> str: return f"running dbt with arguments {str(self.args)}" class MainTrackingUserState(DebugLevel): - def code(self): + def code(self) -> str: return "A003" - def message(self): + def message(self) -> str: return f"Tracking: {self.user_state}" class MergedFromState(DebugLevel): - def code(self): + def code(self) -> str: return "A004" def message(self) -> str: @@ -84,7 +84,7 @@ def message(self) -> str: class MissingProfileTarget(InfoLevel): - def code(self): + def code(self) -> str: return "A005" def message(self) -> str: @@ -95,7 +95,7 @@ def message(self) -> str: class InvalidOptionYAML(ErrorLevel): - def code(self): + def code(self) -> str: return "A008" def message(self) -> str: @@ -103,7 +103,7 @@ def message(self) -> str: class LogDbtProjectError(ErrorLevel): - def code(self): + def code(self) -> str: return "A009" def message(self) -> str: @@ -117,7 +117,7 @@ def message(self) -> str: class LogDbtProfileError(ErrorLevel): - def code(self): + def code(self) -> str: return "A011" def message(self) -> str: @@ -138,7 +138,7 @@ def message(self) -> str: class StarterProjectPath(DebugLevel): - def code(self): + def code(self) -> str: return "A017" def message(self) -> str: @@ -146,7 +146,7 @@ def message(self) -> str: class ConfigFolderDirectory(InfoLevel): - def code(self): + def code(self) -> str: return "A018" def message(self) -> str: @@ -154,7 +154,7 @@ def message(self) -> str: class NoSampleProfileFound(InfoLevel): - def code(self): + def code(self) -> str: return "A019" def message(self) -> str: @@ -162,7 +162,7 @@ def message(self) -> str: class ProfileWrittenWithSample(InfoLevel): - def code(self): + def code(self) -> str: return "A020" def message(self) -> str: @@ -174,7 +174,7 @@ def message(self) -> str: class ProfileWrittenWithTargetTemplateYAML(InfoLevel): - def code(self): + def code(self) -> str: return "A021" def message(self) -> str: @@ -186,7 +186,7 @@ def message(self) -> str: class ProfileWrittenWithProjectTemplateYAML(InfoLevel): - def code(self): + def code(self) -> str: return "A022" def message(self) -> str: @@ -198,7 +198,7 @@ def message(self) -> str: class SettingUpProfile(InfoLevel): - def code(self): + def code(self) -> str: return "A023" def message(self) -> str: @@ -206,7 +206,7 @@ def message(self) -> str: class InvalidProfileTemplateYAML(InfoLevel): - def code(self): + def code(self) -> str: return "A024" def message(self) -> str: @@ -214,7 +214,7 @@ def message(self) -> str: class ProjectNameAlreadyExists(InfoLevel): - def code(self): + def code(self) -> str: return "A025" def message(self) -> str: @@ -222,7 +222,7 @@ def message(self) -> str: class ProjectCreated(InfoLevel): - def code(self): + def code(self) -> str: return "A026" def message(self) -> str: @@ -250,10 +250,10 @@ def message(self) -> str: class PackageRedirectDeprecation(WarnLevel): - def code(self): + def code(self) -> str: return "D001" - def message(self): + def message(self) -> str: description = ( f"The `{self.old_name}` package is deprecated in favor of `{self.new_name}`. Please " f"update your `packages.yml` configuration to use `{self.new_name}` instead." @@ -262,10 +262,10 @@ def message(self): class PackageInstallPathDeprecation(WarnLevel): - def code(self): + def code(self) -> str: return "D002" - def message(self): + def message(self) -> str: description = """\ The default package install path has changed from `dbt_modules` to `dbt_packages`. Please update `clean-targets` in `dbt_project.yml` and check `.gitignore` as well. @@ -275,10 +275,10 @@ def message(self): class ConfigSourcePathDeprecation(WarnLevel): - def code(self): + def code(self) -> str: return "D003" - def message(self): + def message(self) -> str: description = ( f"The `{self.deprecated_path}` config has been renamed to `{self.exp_path}`. " "Please update your `dbt_project.yml` configuration to reflect this change." @@ -287,10 +287,10 @@ def message(self): class ConfigDataPathDeprecation(WarnLevel): - def code(self): + def code(self) -> str: return "D004" - def message(self): + def message(self) -> str: description = ( f"The `{self.deprecated_path}` config has been renamed to `{self.exp_path}`. " "Please update your `dbt_project.yml` configuration to reflect this change." @@ -299,10 +299,10 @@ def message(self): class AdapterDeprecationWarning(WarnLevel): - def code(self): + def code(self) -> str: return "D005" - def message(self): + def message(self) -> str: description = ( f"The adapter function `adapter.{self.old_name}` is deprecated and will be removed in " f"a future release of dbt. Please use `adapter.{self.new_name}` instead. " @@ -313,10 +313,10 @@ def message(self): class MetricAttributesRenamed(WarnLevel): - def code(self): + def code(self) -> str: return "D006" - def message(self): + def message(self) -> str: description = ( "dbt-core v1.3 renamed attributes for metrics:" "\n 'sql' -> 'expression'" @@ -330,10 +330,10 @@ def message(self): class ExposureNameDeprecation(WarnLevel): - def code(self): + def code(self) -> str: return "D007" - def message(self): + def message(self) -> str: description = ( "Starting in v1.3, the 'name' of an exposure should contain only letters, " "numbers, and underscores. Exposures support a new property, 'label', which may " @@ -345,10 +345,10 @@ def message(self): class InternalDeprecation(WarnLevel): - def code(self): + def code(self) -> str: return "D008" - def message(self): + def message(self) -> str: extra_reason = "" if self.reason: extra_reason = f"\n{self.reason}" @@ -360,10 +360,10 @@ def message(self): class EnvironmentVariableRenamed(WarnLevel): - def code(self): + def code(self) -> str: return "D009" - def message(self): + def message(self) -> str: description = ( f"The environment variable `{self.old_name}` has been renamed as `{self.new_name}`.\n" f"If `{self.old_name}` is currently set, its value will be used instead of `{self.new_name}`.\n" @@ -374,10 +374,10 @@ def message(self): class ConfigLogPathDeprecation(WarnLevel): - def code(self): + def code(self) -> str: return "D010" - def message(self): + def message(self) -> str: output = "logs" cli_flag = "--log-path" env_var = "DBT_LOG_PATH" @@ -391,10 +391,10 @@ def message(self): class ConfigTargetPathDeprecation(WarnLevel): - def code(self): + def code(self) -> str: return "D011" - def message(self): + def message(self) -> str: output = "artifacts" cli_flag = "--target-path" env_var = "DBT_TARGET_PATH" @@ -408,10 +408,10 @@ def message(self): class CollectFreshnessReturnSignature(WarnLevel): - def code(self): + def code(self) -> str: return "D012" - def message(self): + def message(self) -> str: description = ( "The 'collect_freshness' macro signature has changed to return the full " "query result, rather than just a table of values. See the v1.5 migration guide " @@ -426,39 +426,39 @@ def message(self): class AdapterEventDebug(DebugLevel): - def code(self): + def code(self) -> str: return "E001" - def message(self): + def message(self) -> str: return format_adapter_message(self.name, self.base_msg, self.args) class AdapterEventInfo(InfoLevel): - def code(self): + def code(self) -> str: return "E002" - def message(self): + def message(self) -> str: return format_adapter_message(self.name, self.base_msg, self.args) class AdapterEventWarning(WarnLevel): - def code(self): + def code(self) -> str: return "E003" - def message(self): + def message(self) -> str: return format_adapter_message(self.name, self.base_msg, self.args) class AdapterEventError(ErrorLevel): - def code(self): + def code(self) -> str: return "E004" - def message(self): + def message(self) -> str: return format_adapter_message(self.name, self.base_msg, self.args) class NewConnection(DebugLevel): - def code(self): + def code(self) -> str: return "E005" def message(self) -> str: @@ -466,7 +466,7 @@ def message(self) -> str: class ConnectionReused(DebugLevel): - def code(self): + def code(self) -> str: return "E006" def message(self) -> str: @@ -474,7 +474,7 @@ def message(self) -> str: class ConnectionLeftOpenInCleanup(DebugLevel): - def code(self): + def code(self) -> str: return "E007" def message(self) -> str: @@ -482,7 +482,7 @@ def message(self) -> str: class ConnectionClosedInCleanup(DebugLevel): - def code(self): + def code(self) -> str: return "E008" def message(self) -> str: @@ -490,7 +490,7 @@ def message(self) -> str: class RollbackFailed(DebugLevel): - def code(self): + def code(self) -> str: return "E009" def message(self) -> str: @@ -498,7 +498,7 @@ def message(self) -> str: class ConnectionClosed(DebugLevel): - def code(self): + def code(self) -> str: return "E010" def message(self) -> str: @@ -506,7 +506,7 @@ def message(self) -> str: class ConnectionLeftOpen(DebugLevel): - def code(self): + def code(self) -> str: return "E011" def message(self) -> str: @@ -514,7 +514,7 @@ def message(self) -> str: class Rollback(DebugLevel): - def code(self): + def code(self) -> str: return "E012" def message(self) -> str: @@ -522,7 +522,7 @@ def message(self) -> str: class CacheMiss(DebugLevel): - def code(self): + def code(self) -> str: return "E013" def message(self) -> str: @@ -533,7 +533,7 @@ def message(self) -> str: class ListRelations(DebugLevel): - def code(self): + def code(self) -> str: return "E014" def message(self) -> str: @@ -542,7 +542,7 @@ def message(self) -> str: class ConnectionUsed(DebugLevel): - def code(self): + def code(self) -> str: return "E015" def message(self) -> str: @@ -550,7 +550,7 @@ def message(self) -> str: class SQLQuery(DebugLevel): - def code(self): + def code(self) -> str: return "E016" def message(self) -> str: @@ -558,7 +558,7 @@ def message(self) -> str: class SQLQueryStatus(DebugLevel): - def code(self): + def code(self) -> str: return "E017" def message(self) -> str: @@ -566,7 +566,7 @@ def message(self) -> str: class SQLCommit(DebugLevel): - def code(self): + def code(self) -> str: return "E018" def message(self) -> str: @@ -574,7 +574,7 @@ def message(self) -> str: class ColTypeChange(DebugLevel): - def code(self): + def code(self) -> str: return "E019" def message(self) -> str: @@ -582,7 +582,7 @@ def message(self) -> str: class SchemaCreation(DebugLevel): - def code(self): + def code(self) -> str: return "E020" def message(self) -> str: @@ -590,7 +590,7 @@ def message(self) -> str: class SchemaDrop(DebugLevel): - def code(self): + def code(self) -> str: return "E021" def message(self) -> str: @@ -598,13 +598,13 @@ def message(self) -> str: class CacheAction(DebugLevel): - def code(self): + def code(self) -> str: return "E022" - def format_ref_key(self, ref_key): + def format_ref_key(self, ref_key) -> str: return f"(database={ref_key.database}, schema={ref_key.schema}, identifier={ref_key.identifier})" - def message(self): + def message(self) -> str: ref_key = self.format_ref_key(self.ref_key) ref_key_2 = self.format_ref_key(self.ref_key_2) ref_key_3 = self.format_ref_key(self.ref_key_3) @@ -644,7 +644,7 @@ def message(self): class CacheDumpGraph(DebugLevel): - def code(self): + def code(self) -> str: return "E031" def message(self) -> str: @@ -655,7 +655,7 @@ def message(self) -> str: class AdapterRegistered(InfoLevel): - def code(self): + def code(self) -> str: return "E034" def message(self) -> str: @@ -663,7 +663,7 @@ def message(self) -> str: class AdapterImportError(InfoLevel): - def code(self): + def code(self) -> str: return "E035" def message(self) -> str: @@ -671,15 +671,15 @@ def message(self) -> str: class PluginLoadError(DebugLevel): - def code(self): + def code(self) -> str: return "E036" - def message(self): + def message(self) -> str: return f"{self.exc_info}" class NewConnectionOpening(DebugLevel): - def code(self): + def code(self) -> str: return "E037" def message(self) -> str: @@ -687,7 +687,7 @@ def message(self) -> str: class CodeExecution(DebugLevel): - def code(self): + def code(self) -> str: return "E038" def message(self) -> str: @@ -695,7 +695,7 @@ def message(self) -> str: class CodeExecutionStatus(DebugLevel): - def code(self): + def code(self) -> str: return "E039" def message(self) -> str: @@ -703,7 +703,7 @@ def message(self) -> str: class CatalogGenerationError(WarnLevel): - def code(self): + def code(self) -> str: return "E040" def message(self) -> str: @@ -711,7 +711,7 @@ def message(self) -> str: class WriteCatalogFailure(ErrorLevel): - def code(self): + def code(self) -> str: return "E041" def message(self) -> str: @@ -722,7 +722,7 @@ def message(self) -> str: class CatalogWritten(InfoLevel): - def code(self): + def code(self) -> str: return "E042" def message(self) -> str: @@ -730,7 +730,7 @@ def message(self) -> str: class CannotGenerateDocs(InfoLevel): - def code(self): + def code(self) -> str: return "E043" def message(self) -> str: @@ -738,7 +738,7 @@ def message(self) -> str: class BuildingCatalog(InfoLevel): - def code(self): + def code(self) -> str: return "E044" def message(self) -> str: @@ -746,7 +746,7 @@ def message(self) -> str: class DatabaseErrorRunningHook(InfoLevel): - def code(self): + def code(self) -> str: return "E045" def message(self) -> str: @@ -754,7 +754,7 @@ def message(self) -> str: class HooksRunning(InfoLevel): - def code(self): + def code(self) -> str: return "E046" def message(self) -> str: @@ -763,7 +763,7 @@ def message(self) -> str: class FinishedRunningStats(InfoLevel): - def code(self): + def code(self) -> str: return "E047" def message(self) -> str: @@ -771,7 +771,7 @@ def message(self) -> str: class ConstraintNotEnforced(WarnLevel): - def code(self): + def code(self) -> str: return "E048" def message(self) -> str: @@ -785,7 +785,7 @@ def message(self) -> str: class ConstraintNotSupported(WarnLevel): - def code(self): + def code(self) -> str: return "E049" def message(self) -> str: @@ -802,7 +802,7 @@ def message(self) -> str: class InputFileDiffError(DebugLevel): - def code(self): + def code(self) -> str: return "I001" def message(self) -> str: @@ -813,7 +813,7 @@ def message(self) -> str: class InvalidValueForField(WarnLevel): - def code(self): + def code(self) -> str: return "I008" def message(self) -> str: @@ -821,7 +821,7 @@ def message(self) -> str: class ValidationWarning(WarnLevel): - def code(self): + def code(self) -> str: return "I009" def message(self) -> str: @@ -829,7 +829,7 @@ def message(self) -> str: class ParsePerfInfoPath(InfoLevel): - def code(self): + def code(self) -> str: return "I010" def message(self) -> str: @@ -846,7 +846,7 @@ def message(self) -> str: class PartialParsingErrorProcessingFile(DebugLevel): - def code(self): + def code(self) -> str: return "I014" def message(self) -> str: @@ -857,7 +857,7 @@ def message(self) -> str: class PartialParsingError(DebugLevel): - def code(self): + def code(self) -> str: return "I016" def message(self) -> str: @@ -865,7 +865,7 @@ def message(self) -> str: class PartialParsingSkipParsing(DebugLevel): - def code(self): + def code(self) -> str: return "I017" def message(self) -> str: @@ -876,7 +876,7 @@ def message(self) -> str: class UnableToPartialParse(InfoLevel): - def code(self): + def code(self) -> str: return "I024" def message(self) -> str: @@ -884,7 +884,7 @@ def message(self) -> str: class StateCheckVarsHash(DebugLevel): - def code(self): + def code(self) -> str: return "I025" def message(self) -> str: @@ -895,7 +895,7 @@ def message(self) -> str: class PartialParsingNotEnabled(DebugLevel): - def code(self): + def code(self) -> str: return "I028" def message(self) -> str: @@ -903,7 +903,7 @@ def message(self) -> str: class ParsedFileLoadFailed(DebugLevel): - def code(self): + def code(self) -> str: return "I029" def message(self) -> str: @@ -914,7 +914,7 @@ def message(self) -> str: class PartialParsingEnabled(DebugLevel): - def code(self): + def code(self) -> str: return "I040" def message(self) -> str: @@ -927,7 +927,7 @@ def message(self) -> str: class PartialParsingFile(DebugLevel): - def code(self): + def code(self) -> str: return "I041" def message(self) -> str: @@ -938,7 +938,7 @@ def message(self) -> str: class InvalidDisabledTargetInTestNode(DebugLevel): - def code(self): + def code(self) -> str: return "I050" def message(self) -> str: @@ -957,7 +957,7 @@ def message(self) -> str: class UnusedResourceConfigPath(WarnLevel): - def code(self): + def code(self) -> str: return "I051" def message(self) -> str: @@ -971,7 +971,7 @@ def message(self) -> str: class SeedIncreased(WarnLevel): - def code(self): + def code(self) -> str: return "I052" def message(self) -> str: @@ -984,7 +984,7 @@ def message(self) -> str: class SeedExceedsLimitSamePath(WarnLevel): - def code(self): + def code(self) -> str: return "I053" def message(self) -> str: @@ -997,7 +997,7 @@ def message(self) -> str: class SeedExceedsLimitAndPathChanged(WarnLevel): - def code(self): + def code(self) -> str: return "I054" def message(self) -> str: @@ -1010,7 +1010,7 @@ def message(self) -> str: class SeedExceedsLimitChecksumChanged(WarnLevel): - def code(self): + def code(self) -> str: return "I055" def message(self) -> str: @@ -1023,7 +1023,7 @@ def message(self) -> str: class UnusedTables(WarnLevel): - def code(self): + def code(self) -> str: return "I056" def message(self) -> str: @@ -1036,7 +1036,7 @@ def message(self) -> str: class WrongResourceSchemaFile(WarnLevel): - def code(self): + def code(self) -> str: return "I057" def message(self) -> str: @@ -1053,7 +1053,7 @@ def message(self) -> str: class NoNodeForYamlKey(WarnLevel): - def code(self): + def code(self) -> str: return "I058" def message(self) -> str: @@ -1066,7 +1066,7 @@ def message(self) -> str: class MacroNotFoundForPatch(WarnLevel): - def code(self): + def code(self) -> str: return "I059" def message(self) -> str: @@ -1075,13 +1075,13 @@ def message(self) -> str: class NodeNotFoundOrDisabled(WarnLevel): - def code(self): + def code(self) -> str: return "I060" def message(self) -> str: # this is duplicated logic from exceptions.get_not_found_or_disabled_msg - # when we convert exceptions to be stuctured maybe it can be combined? - # convverting the bool to a string since None is also valid + # when we convert exceptions to be structured maybe it can be combined? + # converting the bool to a string since None is also valid if self.disabled == "None": reason = "was not found or is disabled" elif self.disabled == "True": @@ -1104,7 +1104,7 @@ def message(self) -> str: class JinjaLogWarning(WarnLevel): - def code(self): + def code(self) -> str: return "I061" def message(self) -> str: @@ -1112,7 +1112,7 @@ def message(self) -> str: class JinjaLogInfo(InfoLevel): - def code(self): + def code(self) -> str: return "I062" def message(self) -> str: @@ -1121,7 +1121,7 @@ def message(self) -> str: class JinjaLogDebug(DebugLevel): - def code(self): + def code(self) -> str: return "I063" def message(self) -> str: @@ -1130,7 +1130,7 @@ def message(self) -> str: class UnpinnedRefNewVersionAvailable(InfoLevel): - def code(self): + def code(self) -> str: return "I064" def message(self) -> str: @@ -1147,7 +1147,7 @@ def message(self) -> str: class DeprecatedModel(WarnLevel): - def code(self): + def code(self) -> str: return "I065" def message(self) -> str: @@ -1160,7 +1160,7 @@ def message(self) -> str: class UpcomingReferenceDeprecation(WarnLevel): - def code(self): + def code(self) -> str: return "I066" def message(self) -> str: @@ -1182,7 +1182,7 @@ def message(self) -> str: class DeprecatedReference(WarnLevel): - def code(self): + def code(self) -> str: return "I067" def message(self) -> str: @@ -1204,7 +1204,7 @@ def message(self) -> str: class UnsupportedConstraintMaterialization(WarnLevel): - def code(self): + def code(self) -> str: return "I068" def message(self) -> str: @@ -1217,7 +1217,7 @@ def message(self) -> str: class ParseInlineNodeError(ErrorLevel): - def code(self): + def code(self) -> str: return "I069" def message(self) -> str: @@ -1225,7 +1225,7 @@ def message(self) -> str: class SemanticValidationFailure(WarnLevel): - def code(self): + def code(self) -> str: return "I070" def message(self) -> str: @@ -1233,7 +1233,7 @@ def message(self) -> str: class UnversionedBreakingChange(WarnLevel): - def code(self): + def code(self) -> str: return "I071" def message(self) -> str: @@ -1263,7 +1263,7 @@ def message(self) -> str: class GitSparseCheckoutSubdirectory(DebugLevel): - def code(self): + def code(self) -> str: return "M001" def message(self) -> str: @@ -1271,7 +1271,7 @@ def message(self) -> str: class GitProgressCheckoutRevision(DebugLevel): - def code(self): + def code(self) -> str: return "M002" def message(self) -> str: @@ -1279,7 +1279,7 @@ def message(self) -> str: class GitProgressUpdatingExistingDependency(DebugLevel): - def code(self): + def code(self) -> str: return "M003" def message(self) -> str: @@ -1287,7 +1287,7 @@ def message(self) -> str: class GitProgressPullingNewDependency(DebugLevel): - def code(self): + def code(self) -> str: return "M004" def message(self) -> str: @@ -1295,7 +1295,7 @@ def message(self) -> str: class GitNothingToDo(DebugLevel): - def code(self): + def code(self) -> str: return "M005" def message(self) -> str: @@ -1303,7 +1303,7 @@ def message(self) -> str: class GitProgressUpdatedCheckoutRange(DebugLevel): - def code(self): + def code(self) -> str: return "M006" def message(self) -> str: @@ -1311,7 +1311,7 @@ def message(self) -> str: class GitProgressCheckedOutAt(DebugLevel): - def code(self): + def code(self) -> str: return "M007" def message(self) -> str: @@ -1319,7 +1319,7 @@ def message(self) -> str: class RegistryProgressGETRequest(DebugLevel): - def code(self): + def code(self) -> str: return "M008" def message(self) -> str: @@ -1327,7 +1327,7 @@ def message(self) -> str: class RegistryProgressGETResponse(DebugLevel): - def code(self): + def code(self) -> str: return "M009" def message(self) -> str: @@ -1335,7 +1335,7 @@ def message(self) -> str: class SelectorReportInvalidSelector(InfoLevel): - def code(self): + def code(self) -> str: return "M010" def message(self) -> str: @@ -1346,7 +1346,7 @@ def message(self) -> str: class DepsNoPackagesFound(InfoLevel): - def code(self): + def code(self) -> str: return "M013" def message(self) -> str: @@ -1354,7 +1354,7 @@ def message(self) -> str: class DepsStartPackageInstall(InfoLevel): - def code(self): + def code(self) -> str: return "M014" def message(self) -> str: @@ -1362,7 +1362,7 @@ def message(self) -> str: class DepsInstallInfo(InfoLevel): - def code(self): + def code(self) -> str: return "M015" def message(self) -> str: @@ -1370,7 +1370,7 @@ def message(self) -> str: class DepsUpdateAvailable(InfoLevel): - def code(self): + def code(self) -> str: return "M016" def message(self) -> str: @@ -1378,7 +1378,7 @@ def message(self) -> str: class DepsUpToDate(InfoLevel): - def code(self): + def code(self) -> str: return "M017" def message(self) -> str: @@ -1386,7 +1386,7 @@ def message(self) -> str: class DepsListSubdirectory(InfoLevel): - def code(self): + def code(self) -> str: return "M018" def message(self) -> str: @@ -1394,7 +1394,7 @@ def message(self) -> str: class DepsNotifyUpdatesAvailable(InfoLevel): - def code(self): + def code(self) -> str: return "M019" def message(self) -> str: @@ -1403,7 +1403,7 @@ def message(self) -> str: class RetryExternalCall(DebugLevel): - def code(self): + def code(self) -> str: return "M020" def message(self) -> str: @@ -1411,7 +1411,7 @@ def message(self) -> str: class RecordRetryException(DebugLevel): - def code(self): + def code(self) -> str: return "M021" def message(self) -> str: @@ -1419,7 +1419,7 @@ def message(self) -> str: class RegistryIndexProgressGETRequest(DebugLevel): - def code(self): + def code(self) -> str: return "M022" def message(self) -> str: @@ -1427,7 +1427,7 @@ def message(self) -> str: class RegistryIndexProgressGETResponse(DebugLevel): - def code(self): + def code(self) -> str: return "M023" def message(self) -> str: @@ -1435,7 +1435,7 @@ def message(self) -> str: class RegistryResponseUnexpectedType(DebugLevel): - def code(self): + def code(self) -> str: return "M024" def message(self) -> str: @@ -1443,7 +1443,7 @@ def message(self) -> str: class RegistryResponseMissingTopKeys(DebugLevel): - def code(self): + def code(self) -> str: return "M025" def message(self) -> str: @@ -1452,7 +1452,7 @@ def message(self) -> str: class RegistryResponseMissingNestedKeys(DebugLevel): - def code(self): + def code(self) -> str: return "M026" def message(self) -> str: @@ -1461,7 +1461,7 @@ def message(self) -> str: class RegistryResponseExtraNestedKeys(DebugLevel): - def code(self): + def code(self) -> str: return "M027" def message(self) -> str: @@ -1470,7 +1470,7 @@ def message(self) -> str: class DepsSetDownloadDirectory(DebugLevel): - def code(self): + def code(self) -> str: return "M028" def message(self) -> str: @@ -1478,7 +1478,7 @@ def message(self) -> str: class DepsUnpinned(WarnLevel): - def code(self): + def code(self) -> str: return "M029" def message(self) -> str: @@ -1497,7 +1497,7 @@ def message(self) -> str: class NoNodesForSelectionCriteria(WarnLevel): - def code(self): + def code(self) -> str: return "M030" def message(self) -> str: @@ -1510,7 +1510,7 @@ def message(self) -> str: class RunningOperationCaughtError(ErrorLevel): - def code(self): + def code(self) -> str: return "Q001" def message(self) -> str: @@ -1518,7 +1518,7 @@ def message(self) -> str: class CompileComplete(InfoLevel): - def code(self): + def code(self) -> str: return "Q002" def message(self) -> str: @@ -1526,7 +1526,7 @@ def message(self) -> str: class FreshnessCheckComplete(InfoLevel): - def code(self): + def code(self) -> str: return "Q003" def message(self) -> str: @@ -1534,7 +1534,7 @@ def message(self) -> str: class SeedHeader(InfoLevel): - def code(self): + def code(self) -> str: return "Q004" def message(self) -> str: @@ -1542,7 +1542,7 @@ def message(self) -> str: class SQLRunnerException(DebugLevel): - def code(self): + def code(self) -> str: return "Q006" def message(self) -> str: @@ -1550,7 +1550,7 @@ def message(self) -> str: class LogTestResult(DynamicLevel): - def code(self): + def code(self) -> str: return "Q007" def message(self) -> str: @@ -1595,7 +1595,7 @@ def status_to_level(cls, status): class LogStartLine(InfoLevel): - def code(self): + def code(self) -> str: return "Q011" def message(self) -> str: @@ -1604,7 +1604,7 @@ def message(self) -> str: class LogModelResult(DynamicLevel): - def code(self): + def code(self) -> str: return "Q012" def message(self) -> str: @@ -1629,7 +1629,7 @@ def message(self) -> str: class LogSnapshotResult(DynamicLevel): - def code(self): + def code(self) -> str: return "Q015" def message(self) -> str: @@ -1651,7 +1651,7 @@ def message(self) -> str: class LogSeedResult(DynamicLevel): - def code(self): + def code(self) -> str: return "Q016" def message(self) -> str: @@ -1675,7 +1675,7 @@ def message(self) -> str: class LogFreshnessResult(DynamicLevel): - def code(self): + def code(self) -> str: return "Q018" def message(self) -> str: @@ -1720,7 +1720,7 @@ def status_to_level(cls, status): class LogCancelLine(ErrorLevel): - def code(self): + def code(self) -> str: return "Q022" def message(self) -> str: @@ -1729,7 +1729,7 @@ def message(self) -> str: class DefaultSelector(InfoLevel): - def code(self): + def code(self) -> str: return "Q023" def message(self) -> str: @@ -1737,7 +1737,7 @@ def message(self) -> str: class NodeStart(DebugLevel): - def code(self): + def code(self) -> str: return "Q024" def message(self) -> str: @@ -1745,7 +1745,7 @@ def message(self) -> str: class NodeFinished(DebugLevel): - def code(self): + def code(self) -> str: return "Q025" def message(self) -> str: @@ -1753,7 +1753,7 @@ def message(self) -> str: class QueryCancelationUnsupported(InfoLevel): - def code(self): + def code(self) -> str: return "Q026" def message(self) -> str: @@ -1766,7 +1766,7 @@ def message(self) -> str: class ConcurrencyLine(InfoLevel): - def code(self): + def code(self) -> str: return "Q027" def message(self) -> str: @@ -1774,7 +1774,7 @@ def message(self) -> str: class WritingInjectedSQLForNode(DebugLevel): - def code(self): + def code(self) -> str: return "Q029" def message(self) -> str: @@ -1782,7 +1782,7 @@ def message(self) -> str: class NodeCompiling(DebugLevel): - def code(self): + def code(self) -> str: return "Q030" def message(self) -> str: @@ -1790,7 +1790,7 @@ def message(self) -> str: class NodeExecuting(DebugLevel): - def code(self): + def code(self) -> str: return "Q031" def message(self) -> str: @@ -1798,7 +1798,7 @@ def message(self) -> str: class LogHookStartLine(InfoLevel): - def code(self): + def code(self) -> str: return "Q032" def message(self) -> str: @@ -1809,7 +1809,7 @@ def message(self) -> str: class LogHookEndLine(InfoLevel): - def code(self): + def code(self) -> str: return "Q033" def message(self) -> str: @@ -1825,7 +1825,7 @@ def message(self) -> str: class SkippingDetails(InfoLevel): - def code(self): + def code(self) -> str: return "Q034" def message(self) -> str: @@ -1839,7 +1839,7 @@ def message(self) -> str: class NothingToDo(WarnLevel): - def code(self): + def code(self) -> str: return "Q035" def message(self) -> str: @@ -1847,7 +1847,7 @@ def message(self) -> str: class RunningOperationUncaughtError(ErrorLevel): - def code(self): + def code(self) -> str: return "Q036" def message(self) -> str: @@ -1855,7 +1855,7 @@ def message(self) -> str: class EndRunResult(DebugLevel): - def code(self): + def code(self) -> str: return "Q037" def message(self) -> str: @@ -1863,7 +1863,7 @@ def message(self) -> str: class NoNodesSelected(WarnLevel): - def code(self): + def code(self) -> str: return "Q038" def message(self) -> str: @@ -1871,7 +1871,7 @@ def message(self) -> str: class CommandCompleted(DebugLevel): - def code(self): + def code(self) -> str: return "Q039" def message(self) -> str: @@ -1881,7 +1881,7 @@ def message(self) -> str: class ShowNode(InfoLevel): - def code(self): + def code(self) -> str: return "Q041" def message(self) -> str: @@ -1900,7 +1900,7 @@ def message(self) -> str: class CompiledNode(InfoLevel): - def code(self): + def code(self) -> str: return "Q042" def message(self) -> str: @@ -1924,7 +1924,7 @@ def message(self) -> str: class CatchableExceptionOnRun(DebugLevel): - def code(self): + def code(self) -> str: return "W002" def message(self) -> str: @@ -1932,7 +1932,7 @@ def message(self) -> str: class InternalErrorOnRun(DebugLevel): - def code(self): + def code(self) -> str: return "W003" def message(self) -> str: @@ -1946,7 +1946,7 @@ def message(self) -> str: class GenericExceptionOnRun(ErrorLevel): - def code(self): + def code(self) -> str: return "W004" def message(self) -> str: @@ -1958,7 +1958,7 @@ def message(self) -> str: class NodeConnectionReleaseError(DebugLevel): - def code(self): + def code(self) -> str: return "W005" def message(self) -> str: @@ -1966,7 +1966,7 @@ def message(self) -> str: class FoundStats(InfoLevel): - def code(self): + def code(self) -> str: return "W006" def message(self) -> str: @@ -1979,7 +1979,7 @@ def message(self) -> str: class MainKeyboardInterrupt(InfoLevel): - def code(self): + def code(self) -> str: return "Z001" def message(self) -> str: @@ -1987,7 +1987,7 @@ def message(self) -> str: class MainEncounteredError(ErrorLevel): - def code(self): + def code(self) -> str: return "Z002" def message(self) -> str: @@ -1995,7 +1995,7 @@ def message(self) -> str: class MainStackTrace(ErrorLevel): - def code(self): + def code(self) -> str: return "Z003" def message(self) -> str: @@ -2006,7 +2006,7 @@ def message(self) -> str: class SystemCouldNotWrite(DebugLevel): - def code(self): + def code(self) -> str: return "Z005" def message(self) -> str: @@ -2017,7 +2017,7 @@ def message(self) -> str: class SystemExecutingCmd(DebugLevel): - def code(self): + def code(self) -> str: return "Z006" def message(self) -> str: @@ -2025,7 +2025,7 @@ def message(self) -> str: class SystemStdOut(DebugLevel): - def code(self): + def code(self) -> str: return "Z007" def message(self) -> str: @@ -2033,7 +2033,7 @@ def message(self) -> str: class SystemStdErr(DebugLevel): - def code(self): + def code(self) -> str: return "Z008" def message(self) -> str: @@ -2041,7 +2041,7 @@ def message(self) -> str: class SystemReportReturnCode(DebugLevel): - def code(self): + def code(self) -> str: return "Z009" def message(self) -> str: @@ -2049,7 +2049,7 @@ def message(self) -> str: class TimingInfoCollected(DebugLevel): - def code(self): + def code(self) -> str: return "Z010" def message(self) -> str: @@ -2063,7 +2063,7 @@ def message(self) -> str: class LogDebugStackTrace(DebugLevel): - def code(self): + def code(self) -> str: return "Z011" def message(self) -> str: @@ -2075,7 +2075,7 @@ def message(self) -> str: class CheckCleanPath(InfoLevel): - def code(self): + def code(self) -> str: return "Z012" def message(self) -> str: @@ -2083,7 +2083,7 @@ def message(self) -> str: class ConfirmCleanPath(InfoLevel): - def code(self): + def code(self) -> str: return "Z013" def message(self) -> str: @@ -2091,7 +2091,7 @@ def message(self) -> str: class ProtectedCleanPath(InfoLevel): - def code(self): + def code(self) -> str: return "Z014" def message(self) -> str: @@ -2099,7 +2099,7 @@ def message(self) -> str: class FinishedCleanPaths(InfoLevel): - def code(self): + def code(self) -> str: return "Z015" def message(self) -> str: @@ -2107,7 +2107,7 @@ def message(self) -> str: class OpenCommand(InfoLevel): - def code(self): + def code(self) -> str: return "Z016" def message(self) -> str: @@ -2126,7 +2126,7 @@ def message(self) -> str: class Formatting(InfoLevel): - def code(self): + def code(self) -> str: return "Z017" def message(self) -> str: @@ -2134,7 +2134,7 @@ def message(self) -> str: class RunResultWarning(WarnLevel): - def code(self): + def code(self) -> str: return "Z021" def message(self) -> str: @@ -2143,7 +2143,7 @@ def message(self) -> str: class RunResultFailure(ErrorLevel): - def code(self): + def code(self) -> str: return "Z022" def message(self) -> str: @@ -2152,7 +2152,7 @@ def message(self) -> str: class StatsLine(InfoLevel): - def code(self): + def code(self) -> str: return "Z023" def message(self) -> str: @@ -2161,7 +2161,7 @@ def message(self) -> str: class RunResultError(ErrorLevel): - def code(self): + def code(self) -> str: return "Z024" def message(self) -> str: @@ -2170,7 +2170,7 @@ def message(self) -> str: class RunResultErrorNoMessage(ErrorLevel): - def code(self): + def code(self) -> str: return "Z025" def message(self) -> str: @@ -2178,7 +2178,7 @@ def message(self) -> str: class SQLCompiledPath(InfoLevel): - def code(self): + def code(self) -> str: return "Z026" def message(self) -> str: @@ -2186,7 +2186,7 @@ def message(self) -> str: class CheckNodeTestFailure(InfoLevel): - def code(self): + def code(self) -> str: return "Z027" def message(self) -> str: @@ -2199,7 +2199,7 @@ def message(self) -> str: class EndOfRunSummary(InfoLevel): - def code(self): + def code(self) -> str: return "Z030" def message(self) -> str: @@ -2220,7 +2220,7 @@ def message(self) -> str: class LogSkipBecauseError(ErrorLevel): - def code(self): + def code(self) -> str: return "Z034" def message(self) -> str: @@ -2234,7 +2234,7 @@ def message(self) -> str: class EnsureGitInstalled(ErrorLevel): - def code(self): + def code(self) -> str: return "Z036" def message(self) -> str: @@ -2246,7 +2246,7 @@ def message(self) -> str: class DepsCreatingLocalSymlink(DebugLevel): - def code(self): + def code(self) -> str: return "Z037" def message(self) -> str: @@ -2254,7 +2254,7 @@ def message(self) -> str: class DepsSymlinkNotAvailable(DebugLevel): - def code(self): + def code(self) -> str: return "Z038" def message(self) -> str: @@ -2262,7 +2262,7 @@ def message(self) -> str: class DisableTracking(DebugLevel): - def code(self): + def code(self) -> str: return "Z039" def message(self) -> str: @@ -2274,7 +2274,7 @@ def message(self) -> str: class SendingEvent(DebugLevel): - def code(self): + def code(self) -> str: return "Z040" def message(self) -> str: @@ -2282,7 +2282,7 @@ def message(self) -> str: class SendEventFailure(DebugLevel): - def code(self): + def code(self) -> str: return "Z041" def message(self) -> str: @@ -2290,7 +2290,7 @@ def message(self) -> str: class FlushEvents(DebugLevel): - def code(self): + def code(self) -> str: return "Z042" def message(self) -> str: @@ -2298,7 +2298,7 @@ def message(self) -> str: class FlushEventsFailure(DebugLevel): - def code(self): + def code(self) -> str: return "Z043" def message(self) -> str: @@ -2306,7 +2306,7 @@ def message(self) -> str: class TrackingInitializeFailure(DebugLevel): - def code(self): + def code(self) -> str: return "Z044" def message(self) -> str: @@ -2317,7 +2317,7 @@ def message(self) -> str: class RunResultWarningMessage(WarnLevel): - def code(self): + def code(self) -> str: return "Z046" def message(self) -> str: @@ -2326,7 +2326,7 @@ def message(self) -> str: class DebugCmdOut(InfoLevel): - def code(self): + def code(self) -> str: return "Z047" def message(self) -> str: @@ -2334,7 +2334,7 @@ def message(self) -> str: class DebugCmdResult(InfoLevel): - def code(self): + def code(self) -> str: return "Z048" def message(self) -> str: @@ -2342,7 +2342,7 @@ def message(self) -> str: class ListCmdOut(InfoLevel): - def code(self): + def code(self) -> str: return "Z049" def message(self) -> str: @@ -2354,7 +2354,7 @@ def message(self) -> str: class Note(InfoLevel): - def code(self): + def code(self) -> str: return "Z050" def message(self) -> str: diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index db97c1d87c8..9646bf5e72e 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -34,7 +34,7 @@ class MacroReturn(builtins.BaseException): It's how we return a value from a macro. """ - def __init__(self, value): + def __init__(self, value) -> None: self.value = value @@ -51,7 +51,7 @@ def data(self): class DbtInternalError(Exception): - def __init__(self, msg: str): + def __init__(self, msg: str) -> None: self.stack: List = [] self.msg = scrub_secrets(msg, env_secrets()) @@ -95,7 +95,7 @@ class DbtRuntimeError(RuntimeError, Exception): CODE = 10001 MESSAGE = "Runtime error" - def __init__(self, msg: str, node=None): + def __init__(self, msg: str, node=None) -> None: self.stack: List = [] self.node = node self.msg = scrub_secrets(msg, env_secrets()) @@ -229,7 +229,7 @@ def __init__( self, breaking_changes: List[str], node=None, - ): + ) -> None: self.breaking_changes = breaking_changes super().__init__(self.message(), node) @@ -273,7 +273,7 @@ class dbtPluginError(DbtRuntimeError): # TODO: this isn't raised in the core codebase. Is it raised elsewhere? class JSONValidationError(DbtValidationError): - def __init__(self, typename, errors): + def __init__(self, typename, errors) -> None: self.typename = typename self.errors = errors self.errors_message = ", ".join(errors) @@ -286,7 +286,7 @@ def __reduce__(self): class IncompatibleSchemaError(DbtRuntimeError): - def __init__(self, expected: str, found: Optional[str] = None): + def __init__(self, expected: str, found: Optional[str] = None) -> None: self.expected = expected self.found = found self.filename = "input file" @@ -340,7 +340,7 @@ class DbtConfigError(DbtRuntimeError): CODE = 10007 MESSAGE = "DBT Configuration Error" - def __init__(self, msg: str, project=None, result_type="invalid_project", path=None): + def __init__(self, msg: str, project=None, result_type="invalid_project", path=None) -> None: self.project = project super().__init__(msg) self.result_type = result_type @@ -358,7 +358,7 @@ class FailFastError(DbtRuntimeError): CODE = 10013 MESSAGE = "FailFast Error" - def __init__(self, msg: str, result=None, node=None): + def __init__(self, msg: str, result=None, node=None) -> None: super().__init__(msg=msg, node=node) self.result = result @@ -380,7 +380,7 @@ class DbtProfileError(DbtConfigError): class SemverError(Exception): - def __init__(self, msg: Optional[str] = None): + def __init__(self, msg: Optional[str] = None) -> None: self.msg = msg if msg is not None: super().__init__(msg) @@ -393,7 +393,7 @@ class VersionsNotCompatibleError(SemverError): class NotImplementedError(Exception): - def __init__(self, msg: str): + def __init__(self, msg: str) -> None: self.msg = msg self.formatted_msg = f"ERROR: {self.msg}" super().__init__(self.formatted_msg) @@ -404,7 +404,7 @@ class FailedToConnectError(DbtDatabaseError): class CommandError(DbtRuntimeError): - def __init__(self, cwd: str, cmd: List[str], msg: str = "Error running command"): + def __init__(self, cwd: str, cmd: List[str], msg: str = "Error running command") -> None: cmd_scrubbed = list(scrub_secrets(cmd_txt, env_secrets()) for cmd_txt in cmd) super().__init__(msg) self.cwd = cwd @@ -418,12 +418,12 @@ def __str__(self): class ExecutableError(CommandError): - def __init__(self, cwd: str, cmd: List[str], msg: str): + def __init__(self, cwd: str, cmd: List[str], msg: str) -> None: super().__init__(cwd, cmd, msg) class WorkingDirectoryError(CommandError): - def __init__(self, cwd: str, cmd: List[str], msg: str): + def __init__(self, cwd: str, cmd: List[str], msg: str) -> None: super().__init__(cwd, cmd, msg) def __str__(self): @@ -439,7 +439,7 @@ def __init__( stdout: bytes, stderr: bytes, msg: str = "Got a non-zero returncode", - ): + ) -> None: super().__init__(cwd, cmd, msg) self.returncode = returncode self.stdout = scrub_secrets(stdout.decode("utf-8"), env_secrets()) @@ -451,7 +451,7 @@ def __str__(self): class InvalidConnectionError(DbtRuntimeError): - def __init__(self, thread_id, known: List): + def __init__(self, thread_id, known: List) -> None: self.thread_id = thread_id self.known = known super().__init__( @@ -460,7 +460,7 @@ def __init__(self, thread_id, known: List): class InvalidSelectorError(DbtRuntimeError): - def __init__(self, name: str): + def __init__(self, name: str) -> None: self.name = name super().__init__(name) @@ -480,7 +480,7 @@ class ConnectionError(Exception): # event level exception class EventCompilationError(CompilationError): - def __init__(self, msg: str, node): + def __init__(self, msg: str, node) -> None: self.msg = scrub_secrets(msg, env_secrets()) self.node = node super().__init__(msg=self.msg) @@ -488,7 +488,7 @@ def __init__(self, msg: str, node): # compilation level exceptions class GraphDependencyNotFoundError(CompilationError): - def __init__(self, node, dependency: str): + def __init__(self, node, dependency: str) -> None: self.node = node self.dependency = dependency super().__init__(msg=self.get_message()) @@ -502,21 +502,21 @@ def get_message(self) -> str: class NoSupportedLanguagesFoundError(CompilationError): - def __init__(self, node): + def __init__(self, node) -> None: self.node = node self.msg = f"No supported_languages found in materialization macro {self.node.name}" super().__init__(msg=self.msg) class MaterializtionMacroNotUsedError(CompilationError): - def __init__(self, node): + def __init__(self, node) -> None: self.node = node self.msg = "Only materialization macros can be used with this function" super().__init__(msg=self.msg) class UndefinedCompilationError(CompilationError): - def __init__(self, name: str, node): + def __init__(self, name: str, node) -> None: self.name = name self.node = node self.msg = f"{self.name} is undefined" @@ -524,20 +524,20 @@ def __init__(self, name: str, node): class CaughtMacroErrorWithNodeError(CompilationError): - def __init__(self, exc, node): + def __init__(self, exc, node) -> None: self.exc = exc self.node = node super().__init__(msg=str(exc)) class CaughtMacroError(CompilationError): - def __init__(self, exc): + def __init__(self, exc) -> None: self.exc = exc super().__init__(msg=str(exc)) class MacroNameNotStringError(CompilationError): - def __init__(self, kwarg_value): + def __init__(self, kwarg_value) -> None: self.kwarg_value = kwarg_value super().__init__(msg=self.get_message()) @@ -550,7 +550,7 @@ def get_message(self) -> str: class MissingControlFlowStartTagError(CompilationError): - def __init__(self, tag, expected_tag: str, tag_parser): + def __init__(self, tag, expected_tag: str, tag_parser) -> None: self.tag = tag self.expected_tag = expected_tag self.tag_parser = tag_parser @@ -566,7 +566,7 @@ def get_message(self) -> str: class UnexpectedControlFlowEndTagError(CompilationError): - def __init__(self, tag, expected_tag: str, tag_parser): + def __init__(self, tag, expected_tag: str, tag_parser) -> None: self.tag = tag self.expected_tag = expected_tag self.tag_parser = tag_parser @@ -582,7 +582,7 @@ def get_message(self) -> str: class UnexpectedMacroEOFError(CompilationError): - def __init__(self, expected_name: str, actual_name: str): + def __init__(self, expected_name: str, actual_name: str) -> None: self.expected_name = expected_name self.actual_name = actual_name super().__init__(msg=self.get_message()) @@ -593,7 +593,7 @@ def get_message(self) -> str: class MacroNamespaceNotStringError(CompilationError): - def __init__(self, kwarg_type: Any): + def __init__(self, kwarg_type: Any) -> None: self.kwarg_type = kwarg_type super().__init__(msg=self.get_message()) @@ -606,7 +606,7 @@ def get_message(self) -> str: class NestedTagsError(CompilationError): - def __init__(self, outer, inner): + def __init__(self, outer, inner) -> None: self.outer = outer self.inner = inner super().__init__(msg=self.get_message()) @@ -621,7 +621,7 @@ def get_message(self) -> str: class BlockDefinitionNotAtTopError(CompilationError): - def __init__(self, tag_parser, tag_start): + def __init__(self, tag_parser, tag_start) -> None: self.tag_parser = tag_parser self.tag_start = tag_start super().__init__(msg=self.get_message()) @@ -636,7 +636,7 @@ def get_message(self) -> str: class MissingCloseTagError(CompilationError): - def __init__(self, block_type_name: str, linecount: int): + def __init__(self, block_type_name: str, linecount: int) -> None: self.block_type_name = block_type_name self.linecount = linecount super().__init__(msg=self.get_message()) @@ -647,7 +647,7 @@ def get_message(self) -> str: class UnknownGitCloningProblemError(DbtRuntimeError): - def __init__(self, repo: str): + def __init__(self, repo: str) -> None: self.repo = scrub_secrets(repo, env_secrets()) super().__init__(msg=self.get_message()) @@ -660,7 +660,7 @@ def get_message(self) -> str: class NoAdaptersAvailableError(DbtRuntimeError): - def __init__(self): + def __init__(self) -> None: super().__init__(msg=self.get_message()) def get_message(self) -> str: @@ -669,7 +669,7 @@ def get_message(self) -> str: class BadSpecError(DbtInternalError): - def __init__(self, repo, revision, error): + def __init__(self, repo, revision, error) -> None: self.repo = repo self.revision = revision self.stderr = scrub_secrets(error.stderr.strip(), env_secrets()) @@ -681,7 +681,7 @@ def get_message(self) -> str: class GitCloningError(DbtInternalError): - def __init__(self, repo: str, revision: str, error: CommandResultError): + def __init__(self, repo: str, revision: str, error: CommandResultError) -> None: self.repo = repo self.revision = revision self.error = error @@ -704,7 +704,7 @@ class GitCheckoutError(BadSpecError): class MaterializationArgError(CompilationError): - def __init__(self, name: str, argument: str): + def __init__(self, name: str, argument: str) -> None: self.name = name self.argument = argument super().__init__(msg=self.get_message()) @@ -715,7 +715,7 @@ def get_message(self) -> str: class OperationError(CompilationError): - def __init__(self, operation_name): + def __init__(self, operation_name) -> None: self.operation_name = operation_name super().__init__(msg=self.get_message()) @@ -730,7 +730,7 @@ def get_message(self) -> str: class SymbolicLinkError(CompilationError): - def __init__(self): + def __init__(self) -> None: super().__init__(msg=self.get_message()) def get_message(self) -> str: @@ -745,21 +745,21 @@ def get_message(self) -> str: # context level exceptions class ZipStrictWrongTypeError(CompilationError): - def __init__(self, exc): + def __init__(self, exc) -> None: self.exc = exc msg = str(self.exc) super().__init__(msg=msg) class SetStrictWrongTypeError(CompilationError): - def __init__(self, exc): + def __init__(self, exc) -> None: self.exc = exc msg = str(self.exc) super().__init__(msg=msg) class LoadAgateTableValueError(CompilationError): - def __init__(self, exc: ValueError, node): + def __init__(self, exc: ValueError, node) -> None: self.exc = exc self.node = node msg = str(self.exc) @@ -767,7 +767,7 @@ def __init__(self, exc: ValueError, node): class LoadAgateTableNotSeedError(CompilationError): - def __init__(self, resource_type, node): + def __init__(self, resource_type, node) -> None: self.resource_type = resource_type self.node = node msg = f"can only load_agate_table for seeds (got a {self.resource_type})" @@ -775,14 +775,14 @@ def __init__(self, resource_type, node): class MacrosSourcesUnWriteableError(CompilationError): - def __init__(self, node): + def __init__(self, node) -> None: self.node = node msg = 'cannot "write" macros or sources' super().__init__(msg=msg) class PackageNotInDepsError(CompilationError): - def __init__(self, package_name: str, node): + def __init__(self, package_name: str, node) -> None: self.package_name = package_name self.node = node msg = f"Node package named {self.package_name} not found!" @@ -790,7 +790,7 @@ def __init__(self, package_name: str, node): class OperationsCannotRefEphemeralNodesError(CompilationError): - def __init__(self, target_name: str, node): + def __init__(self, target_name: str, node) -> None: self.target_name = target_name self.node = node msg = f"Operations can not ref() ephemeral nodes, but {target_name} is ephemeral" @@ -798,7 +798,7 @@ def __init__(self, target_name: str, node): class PersistDocsValueTypeError(CompilationError): - def __init__(self, persist_docs: Any): + def __init__(self, persist_docs: Any) -> None: self.persist_docs = persist_docs msg = ( "Invalid value provided for 'persist_docs'. Expected dict " @@ -808,14 +808,14 @@ def __init__(self, persist_docs: Any): class InlineModelConfigError(CompilationError): - def __init__(self, node): + def __init__(self, node) -> None: self.node = node msg = "Invalid inline model config" super().__init__(msg=msg) class ConflictingConfigKeysError(CompilationError): - def __init__(self, oldkey: str, newkey: str, node): + def __init__(self, oldkey: str, newkey: str, node) -> None: self.oldkey = oldkey self.newkey = newkey self.node = node @@ -824,7 +824,7 @@ def __init__(self, oldkey: str, newkey: str, node): class NumberSourceArgsError(CompilationError): - def __init__(self, args, node): + def __init__(self, args, node) -> None: self.args = args self.node = node msg = f"source() takes exactly two arguments ({len(self.args)} given)" @@ -832,7 +832,7 @@ def __init__(self, args, node): class RequiredVarNotFoundError(CompilationError): - def __init__(self, var_name: str, merged: Dict, node): + def __init__(self, var_name: str, merged: Dict, node) -> None: self.var_name = var_name self.merged = merged self.node = node @@ -852,14 +852,14 @@ def get_message(self) -> str: class PackageNotFoundForMacroError(CompilationError): - def __init__(self, package_name: str): + def __init__(self, package_name: str) -> None: self.package_name = package_name msg = f"Could not find package '{self.package_name}'" super().__init__(msg=msg) class SecretEnvVarLocationError(ParsingError): - def __init__(self, env_var_name: str): + def __init__(self, env_var_name: str) -> None: self.env_var_name = env_var_name super().__init__(msg=self.get_message()) @@ -872,7 +872,7 @@ def get_message(self) -> str: class MacroArgTypeError(CompilationError): - def __init__(self, method_name: str, arg_name: str, got_value: Any, expected_type): + def __init__(self, method_name: str, arg_name: str, got_value: Any, expected_type) -> None: self.method_name = method_name self.arg_name = arg_name self.got_value = got_value @@ -890,7 +890,7 @@ def get_message(self) -> str: class BooleanError(CompilationError): - def __init__(self, return_value: Any, macro_name: str): + def __init__(self, return_value: Any, macro_name: str) -> None: self.return_value = return_value self.macro_name = macro_name super().__init__(msg=self.get_message()) @@ -904,7 +904,7 @@ def get_message(self) -> str: class RefArgsError(CompilationError): - def __init__(self, node, args): + def __init__(self, node, args) -> None: self.node = node self.args = args super().__init__(msg=self.get_message()) @@ -915,7 +915,7 @@ def get_message(self) -> str: class MetricArgsError(CompilationError): - def __init__(self, node, args): + def __init__(self, node, args) -> None: self.node = node self.args = args super().__init__(msg=self.get_message()) @@ -926,7 +926,7 @@ def get_message(self) -> str: class RefBadContextError(CompilationError): - def __init__(self, node, args): + def __init__(self, node, args) -> None: self.node = node self.args = args.positional_args # type: ignore self.kwargs = args.keyword_args # type: ignore @@ -964,7 +964,7 @@ def get_message(self) -> str: class DocArgsError(CompilationError): - def __init__(self, node, args): + def __init__(self, node, args) -> None: self.node = node self.args = args super().__init__(msg=self.get_message()) @@ -975,7 +975,9 @@ def get_message(self) -> str: class DocTargetNotFoundError(CompilationError): - def __init__(self, node, target_doc_name: str, target_doc_package: Optional[str] = None): + def __init__( + self, node, target_doc_name: str, target_doc_package: Optional[str] = None + ) -> None: self.node = node self.target_doc_name = target_doc_name self.target_doc_package = target_doc_package @@ -990,7 +992,7 @@ def get_message(self) -> str: class MacroDispatchArgError(CompilationError): - def __init__(self, macro_name: str): + def __init__(self, macro_name: str) -> None: self.macro_name = macro_name super().__init__(msg=self.get_message()) @@ -1009,7 +1011,7 @@ def get_message(self) -> str: class DuplicateMacroNameError(CompilationError): - def __init__(self, node_1, node_2, namespace: str): + def __init__(self, node_1, node_2, namespace: str) -> None: self.node_1 = node_1 self.node_2 = node_2 self.namespace = namespace @@ -1034,7 +1036,7 @@ def get_message(self) -> str: class MacroResultAlreadyLoadedError(CompilationError): - def __init__(self, result_name): + def __init__(self, result_name) -> None: self.result_name = result_name super().__init__(msg=self.get_message()) @@ -1046,7 +1048,7 @@ def get_message(self) -> str: # parser level exceptions class DictParseError(ParsingError): - def __init__(self, exc: ValidationError, node): + def __init__(self, exc: ValidationError, node) -> None: self.exc = exc self.node = node msg = self.validator_error_message(exc) @@ -1054,7 +1056,7 @@ def __init__(self, exc: ValidationError, node): class ConfigUpdateError(ParsingError): - def __init__(self, exc: ValidationError, node): + def __init__(self, exc: ValidationError, node) -> None: self.exc = exc self.node = node msg = self.validator_error_message(exc) @@ -1062,7 +1064,7 @@ def __init__(self, exc: ValidationError, node): class PythonParsingError(ParsingError): - def __init__(self, exc: SyntaxError, node): + def __init__(self, exc: SyntaxError, node) -> None: self.exc = exc self.node = node super().__init__(msg=self.get_message()) @@ -1074,7 +1076,7 @@ def get_message(self) -> str: class PythonLiteralEvalError(ParsingError): - def __init__(self, exc: Exception, node): + def __init__(self, exc: Exception, node) -> None: self.exc = exc self.node = node super().__init__(msg=self.get_message()) @@ -1090,7 +1092,7 @@ def get_message(self) -> str: class ModelConfigError(ParsingError): - def __init__(self, exc: ValidationError, node): + def __init__(self, exc: ValidationError, node) -> None: self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) @@ -1103,7 +1105,7 @@ def __init__( key: str, yaml_data: List, cause, - ): + ) -> None: self.path = path self.key = key self.yaml_data = yaml_data @@ -1128,7 +1130,7 @@ def __init__( key: str, yaml_data: Dict[str, Any], cause, - ): + ) -> None: self.path = path self.key = key self.yaml_data = yaml_data @@ -1152,7 +1154,7 @@ def __init__( path: str, exc: DbtValidationError, project_name: Optional[str] = None, - ): + ) -> None: self.project_name = project_name self.path = path self.exc = exc @@ -1167,28 +1169,28 @@ def get_message(self) -> str: class TestConfigError(ParsingError): - def __init__(self, exc: ValidationError, node): + def __init__(self, exc: ValidationError, node) -> None: self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) class SchemaConfigError(ParsingError): - def __init__(self, exc: ValidationError, node): + def __init__(self, exc: ValidationError, node) -> None: self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) class SnapshopConfigError(ParsingError): - def __init__(self, exc: ValidationError, node): + def __init__(self, exc: ValidationError, node) -> None: self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) class DbtReferenceError(ParsingError): - def __init__(self, unique_id: str, ref_unique_id: str, access: AccessType, scope: str): + def __init__(self, unique_id: str, ref_unique_id: str, access: AccessType, scope: str) -> None: self.unique_id = unique_id self.ref_unique_id = ref_unique_id self.access = access @@ -1204,7 +1206,9 @@ def get_message(self) -> str: class InvalidAccessTypeError(ParsingError): - def __init__(self, unique_id: str, field_value: str, materialization: Optional[str] = None): + def __init__( + self, unique_id: str, field_value: str, materialization: Optional[str] = None + ) -> None: self.unique_id = unique_id self.field_value = field_value self.materialization = materialization @@ -1217,19 +1221,19 @@ def __init__(self, unique_id: str, field_value: str, materialization: Optional[s class SameKeyNestedError(CompilationError): - def __init__(self): + def __init__(self) -> None: msg = "Test cannot have the same key at the top-level and in config" super().__init__(msg=msg) class TestArgIncludesModelError(CompilationError): - def __init__(self): + def __init__(self) -> None: msg = 'Test arguments include "model", which is a reserved argument' super().__init__(msg=msg) class UnexpectedTestNamePatternError(CompilationError): - def __init__(self, test_name: str): + def __init__(self, test_name: str) -> None: self.test_name = test_name msg = f"Test name string did not match expected pattern: {self.test_name}" super().__init__(msg=msg) @@ -1243,7 +1247,7 @@ def __init__( key: str, err_msg: str, column_name: Optional[str] = None, - ): + ) -> None: self.target_name = target_name self.column_name = column_name self.name = name @@ -1273,21 +1277,21 @@ def get_message(self) -> str: class TagsNotListOfStringsError(CompilationError): - def __init__(self, tags: Any): + def __init__(self, tags: Any) -> None: self.tags = tags msg = f"got {self.tags} ({type(self.tags)}) for tags, expected a list of strings" super().__init__(msg=msg) class TagNotStringError(CompilationError): - def __init__(self, tag: Any): + def __init__(self, tag: Any) -> None: self.tag = tag msg = f"got {self.tag} ({type(self.tag)}) for tag, expected a str" super().__init__(msg=msg) class TestNameNotStringError(ParsingError): - def __init__(self, test_name: Any): + def __init__(self, test_name: Any) -> None: self.test_name = test_name super().__init__(msg=self.get_message()) @@ -1298,7 +1302,7 @@ def get_message(self) -> str: class TestArgsNotDictError(ParsingError): - def __init__(self, test_args: Any): + def __init__(self, test_args: Any) -> None: self.test_args = test_args super().__init__(msg=self.get_message()) diff --git a/core/dbt/graph/graph.py b/core/dbt/graph/graph.py index 69a2f21258a..3bacba1ffc1 100644 --- a/core/dbt/graph/graph.py +++ b/core/dbt/graph/graph.py @@ -12,7 +12,7 @@ class Graph: and how they interact with the graph. """ - def __init__(self, graph): + def __init__(self, graph) -> None: self.graph = graph def nodes(self) -> Set[UniqueId]: diff --git a/core/dbt/graph/queue.py b/core/dbt/graph/queue.py index a21a9afc630..64b8c8438ab 100644 --- a/core/dbt/graph/queue.py +++ b/core/dbt/graph/queue.py @@ -24,7 +24,7 @@ class GraphQueue: the same time, as there is an unlocked race! """ - def __init__(self, graph: nx.DiGraph, manifest: Manifest, selected: Set[UniqueId]): + def __init__(self, graph: nx.DiGraph, manifest: Manifest, selected: Set[UniqueId]) -> None: self.graph = graph self.manifest = manifest self._selected = selected diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index f205fa0fb7a..aacc70550d6 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -44,7 +44,7 @@ def __init__( manifest: Manifest, previous_state: Optional[PreviousState] = None, include_empty_nodes: bool = False, - ): + ) -> None: super().__init__(manifest, previous_state) self.full_graph = graph self.include_empty_nodes = include_empty_nodes @@ -325,7 +325,7 @@ def __init__( previous_state: Optional[PreviousState], resource_types: List[NodeType], include_empty_nodes: bool = False, - ): + ) -> None: super().__init__( graph=graph, manifest=manifest, diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index e2a5a1f3f61..20e78d1c686 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -767,7 +767,7 @@ def __init__( self, manifest: Manifest, previous_state: Optional[PreviousState], - ): + ) -> None: self.manifest = manifest self.previous_state = previous_state diff --git a/core/dbt/graph/selector_spec.py b/core/dbt/graph/selector_spec.py index cf8481ccf65..31ffc050585 100644 --- a/core/dbt/graph/selector_spec.py +++ b/core/dbt/graph/selector_spec.py @@ -176,7 +176,7 @@ def __init__( indirect_selection: IndirectSelection = IndirectSelection.Eager, expect_exists: bool = False, raw: Any = None, - ): + ) -> None: self.components: List[SelectionSpec] = list(components) self.expect_exists = expect_exists self.raw = raw diff --git a/core/dbt/include/global_project/macros/utils/date_spine.sql b/core/dbt/include/global_project/macros/utils/date_spine.sql new file mode 100644 index 00000000000..833fbcc575b --- /dev/null +++ b/core/dbt/include/global_project/macros/utils/date_spine.sql @@ -0,0 +1,75 @@ +{% macro get_intervals_between(start_date, end_date, datepart) -%} + {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }} +{%- endmacro %} + +{% macro default__get_intervals_between(start_date, end_date, datepart) -%} + {%- call statement('get_intervals_between', fetch_result=True) %} + + select {{ dbt.datediff(start_date, end_date, datepart) }} + + {%- endcall -%} + + {%- set value_list = load_result('get_intervals_between') -%} + + {%- if value_list and value_list['data'] -%} + {%- set values = value_list['data'] | map(attribute=0) | list %} + {{ return(values[0]) }} + {%- else -%} + {{ return(1) }} + {%- endif -%} + +{%- endmacro %} + + + + +{% macro date_spine(datepart, start_date, end_date) %} + {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }} +{%- endmacro %} + +{% macro default__date_spine(datepart, start_date, end_date) %} + + + {# call as follows: + + date_spine( + "day", + "to_date('01/01/2016', 'mm/dd/yyyy')", + "dbt.dateadd(week, 1, current_date)" + ) #} + + + with rawdata as ( + + {{dbt.generate_series( + dbt.get_intervals_between(start_date, end_date, datepart) + )}} + + ), + + all_periods as ( + + select ( + {{ + dbt.dateadd( + datepart, + "row_number() over (order by 1) - 1", + start_date + ) + }} + ) as date_{{datepart}} + from rawdata + + ), + + filtered as ( + + select * + from all_periods + where date_{{datepart}} <= {{ end_date }} + + ) + + select * from filtered + +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/utils/generate_series.sql b/core/dbt/include/global_project/macros/utils/generate_series.sql new file mode 100644 index 00000000000..f6a09605af3 --- /dev/null +++ b/core/dbt/include/global_project/macros/utils/generate_series.sql @@ -0,0 +1,53 @@ +{% macro get_powers_of_two(upper_bound) %} + {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }} +{% endmacro %} + +{% macro default__get_powers_of_two(upper_bound) %} + + {% if upper_bound <= 0 %} + {{ exceptions.raise_compiler_error("upper bound must be positive") }} + {% endif %} + + {% for _ in range(1, 100) %} + {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} + {% endfor %} + +{% endmacro %} + + +{% macro generate_series(upper_bound) %} + {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }} +{% endmacro %} + +{% macro default__generate_series(upper_bound) %} + + {% set n = dbt.get_powers_of_two(upper_bound) %} + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + {% for i in range(n) %} + p{{i}}.generated_number * power(2, {{i}}) + {% if not loop.last %} + {% endif %} + {% endfor %} + + 1 + as generated_number + + from + + {% for i in range(n) %} + p as p{{i}} + {% if not loop.last %} cross join {% endif %} + {% endfor %} + + ) + + select * + from unioned + where generated_number <= {{upper_bound}} + order by generated_number + +{% endmacro %} diff --git a/core/dbt/logger.py b/core/dbt/logger.py index d4095fb73bf..2ac90984458 100644 --- a/core/dbt/logger.py +++ b/core/dbt/logger.py @@ -92,7 +92,7 @@ def __call__(self, record, handler): class FormatterMixin: - def __init__(self, format_string): + def __init__(self, format_string) -> None: self._text_format_string = format_string self.formatter_class = logbook.StringFormatter # triggers a formatter update via logbook.StreamHandler @@ -197,7 +197,7 @@ def process(self, record): class TimingProcessor(logbook.Processor): - def __init__(self, timing_info: Optional[dbtClassMixin] = None): + def __init__(self, timing_info: Optional[dbtClassMixin] = None) -> None: self.timing_info = timing_info super().__init__() @@ -207,7 +207,7 @@ def process(self, record): class DbtProcessState(logbook.Processor): - def __init__(self, value: str): + def __init__(self, value: str) -> None: self.value = value super().__init__() @@ -218,7 +218,7 @@ def process(self, record): class DbtModelState(logbook.Processor): - def __init__(self, state: Dict[str, str]): + def __init__(self, state: Dict[str, str]) -> None: self.state = state super().__init__() @@ -232,7 +232,7 @@ def process(self, record): class UniqueID(logbook.Processor): - def __init__(self, unique_id: str): + def __init__(self, unique_id: str) -> None: self.unique_id = unique_id super().__init__() @@ -241,7 +241,7 @@ def process(self, record): class NodeCount(logbook.Processor): - def __init__(self, node_count: int): + def __init__(self, node_count: int) -> None: self.node_count = node_count super().__init__() @@ -250,7 +250,7 @@ def process(self, record): class NodeMetadata(logbook.Processor): - def __init__(self, node, index): + def __init__(self, node, index) -> None: self.node = node self.index = index super().__init__() @@ -301,7 +301,7 @@ def mapping_keys(self): class TimestampNamed(logbook.Processor): - def __init__(self, name: str): + def __init__(self, name: str) -> None: self.name = name super().__init__() @@ -348,7 +348,7 @@ def make_record(self, message, exception, filename, lineno): class LogManager(logbook.NestedSetup): - def __init__(self, stdout=sys.stdout, stderr=sys.stderr): + def __init__(self, stdout=sys.stdout, stderr=sys.stderr) -> None: self.stdout = stdout self.stderr = stderr self._null_handler = logbook.NullHandler() diff --git a/core/dbt/parser/models.py b/core/dbt/parser/models.py index 4fe6cd56082..bff46e41c9f 100644 --- a/core/dbt/parser/models.py +++ b/core/dbt/parser/models.py @@ -33,9 +33,9 @@ class PythonValidationVisitor(ast.NodeVisitor): - def __init__(self): + def __init__(self) -> None: super().__init__() - self.dbt_errors = [] + self.dbt_errors: List[str] = [] self.num_model_def = 0 def visit_FunctionDef(self, node: ast.FunctionDef) -> None: diff --git a/core/dbt/parser/partial.py b/core/dbt/parser/partial.py index 3c6bc46bd74..b337e9e5369 100644 --- a/core/dbt/parser/partial.py +++ b/core/dbt/parser/partial.py @@ -68,7 +68,9 @@ # to preserve an unchanged file object in case we need to drop back to a # a full parse (such as for certain macro changes) class PartialParsing: - def __init__(self, saved_manifest: Manifest, new_files: MutableMapping[str, AnySourceFile]): + def __init__( + self, saved_manifest: Manifest, new_files: MutableMapping[str, AnySourceFile] + ) -> None: self.saved_manifest = saved_manifest self.new_files = new_files self.project_parser_files: Dict = {} diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 032b3a8b99d..dddad84c6db 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -55,7 +55,7 @@ class ExposureParser(YamlReader): - def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock): + def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, NodeType.Exposure.pluralize()) self.schema_parser = schema_parser self.yaml = yaml @@ -158,7 +158,7 @@ def parse(self): class MetricParser(YamlReader): - def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock): + def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, NodeType.Metric.pluralize()) self.schema_parser = schema_parser self.yaml = yaml @@ -178,6 +178,8 @@ def _get_input_measure( name=unparsed_input_measure.name, filter=filter, alias=unparsed_input_measure.alias, + join_to_timespine=unparsed_input_measure.join_to_timespine, + fill_nulls_with=unparsed_input_measure.fill_nulls_with, ) def _get_optional_input_measure( @@ -398,7 +400,7 @@ def parse(self): class GroupParser(YamlReader): - def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock): + def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, NodeType.Group.pluralize()) self.schema_parser = schema_parser self.yaml = yaml @@ -432,7 +434,7 @@ def parse(self): class SemanticModelParser(YamlReader): - def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock): + def __init__(self, schema_parser: SchemaParser, yaml: YamlBlock) -> None: super().__init__(schema_parser, yaml, "semantic_models") self.schema_parser = schema_parser self.yaml = yaml @@ -456,6 +458,7 @@ def _get_dimensions(self, unparsed_dimensions: List[UnparsedDimension]) -> List[ name=unparsed.name, type=DimensionType(unparsed.type), description=unparsed.description, + label=unparsed.label, is_partition=unparsed.is_partition, type_params=self._get_dimension_type_params(unparsed=unparsed.type_params), expr=unparsed.expr, @@ -472,6 +475,7 @@ def _get_entities(self, unparsed_entities: List[UnparsedEntity]) -> List[Entity] name=unparsed.name, type=EntityType(unparsed.type), description=unparsed.description, + label=unparsed.label, role=unparsed.role, expr=unparsed.expr, ) @@ -499,6 +503,7 @@ def _get_measures(self, unparsed_measures: List[UnparsedMeasure]) -> List[Measur name=unparsed.name, agg=AggregationType(unparsed.agg), description=unparsed.description, + label=unparsed.label, expr=str(unparsed.expr) if unparsed.expr is not None else None, agg_params=unparsed.agg_params, non_additive_dimension=self._get_non_additive_dimension( @@ -572,6 +577,7 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel): parsed = SemanticModel( description=unparsed.description, + label=unparsed.label, fqn=fqn, model=unparsed.model, name=unparsed.name, diff --git a/core/dbt/plugins/manager.py b/core/dbt/plugins/manager.py index df6c455528a..cc71d5ab6bf 100644 --- a/core/dbt/plugins/manager.py +++ b/core/dbt/plugins/manager.py @@ -26,7 +26,7 @@ class dbtPlugin: Its interface is **not** stable and will likely change between dbt-core versions. """ - def __init__(self, project_name: str): + def __init__(self, project_name: str) -> None: self.project_name = project_name try: self.initialize() @@ -67,7 +67,7 @@ class PluginManager: PLUGIN_MODULE_PREFIX = "dbt_" PLUGIN_ATTR_NAME = "plugins" - def __init__(self, plugins: List[dbtPlugin]): + def __init__(self, plugins: List[dbtPlugin]) -> None: self._plugins = plugins self._valid_hook_names = set() # default hook implementations from dbtPlugin diff --git a/core/dbt/semver.py b/core/dbt/semver.py index 2afcf25aee7..a1c6a25b929 100644 --- a/core/dbt/semver.py +++ b/core/dbt/semver.py @@ -353,7 +353,7 @@ def to_version_string_pair(self): class UnboundedVersionSpecifier(VersionSpecifier): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__( matcher=Matchers.EXACT, major=None, minor=None, patch=None, prerelease=None, build=None ) diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 3e7d7544578..dcffce7b454 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -5,7 +5,7 @@ from abc import ABCMeta, abstractmethod from contextlib import nullcontext from datetime import datetime -from typing import Type, Union, Dict, Any, Optional +from typing import Any, Dict, List, Optional, Type, Union import dbt.exceptions from dbt import tracking @@ -19,6 +19,7 @@ collect_timing_info, RunStatus, RunningStatus, + TimingInfo, ) from dbt.events.contextvars import get_node_info from dbt.events.functions import fire_event @@ -71,7 +72,7 @@ def read_profiles(profiles_dir=None): class BaseTask(metaclass=ABCMeta): ConfigType: Union[Type[NoneConfig], Type[Project]] = NoneConfig - def __init__(self, args, config, project=None): + def __init__(self, args, config, project=None) -> None: self.args = args self.config = config self.project = config if isinstance(config, Project) else project @@ -156,7 +157,7 @@ def move_to_nearest_project_dir(project_dir: Optional[str]) -> str: class ConfiguredTask(BaseTask): ConfigType = RuntimeConfig - def __init__(self, args, config, manifest: Optional[Manifest] = None): + def __init__(self, args, config, manifest: Optional[Manifest] = None) -> None: super().__init__(args, config) self.graph: Optional[Graph] = None self.manifest = manifest @@ -187,8 +188,8 @@ class ExecutionContext: timing information and the newest (compiled vs executed) form of the node. """ - def __init__(self, node): - self.timing = [] + def __init__(self, node) -> None: + self.timing: List[TimingInfo] = [] self.node = node diff --git a/core/dbt/task/deps.py b/core/dbt/task/deps.py index 849c17b5779..19c80926c31 100644 --- a/core/dbt/task/deps.py +++ b/core/dbt/task/deps.py @@ -28,7 +28,7 @@ class DepsTask(BaseTask): - def __init__(self, args: Any, project: Project): + def __init__(self, args: Any, project: Project) -> None: # N.B. This is a temporary fix for a bug when using relative paths via # --project-dir with deps. A larger overhaul of our path handling methods # is needed to fix this the "right" way. diff --git a/core/dbt/task/generate.py b/core/dbt/task/generate.py index 5e21213e8fb..238b049aa32 100644 --- a/core/dbt/task/generate.py +++ b/core/dbt/task/generate.py @@ -63,7 +63,7 @@ def build_catalog_table(data) -> CatalogTable: # keys are database name, schema name, table name class Catalog(Dict[CatalogKey, CatalogTable]): - def __init__(self, columns: List[PrimitiveDict]): + def __init__(self, columns: List[PrimitiveDict]) -> None: super().__init__() for col in columns: self.add_column(col) diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index 94cd4997cc2..b1edf2b518d 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -46,7 +46,7 @@ class ListTask(GraphRunnableTask): ) ) - def __init__(self, args, config, manifest): + def __init__(self, args, config, manifest) -> None: super().__init__(args, config, manifest) if self.args.models: if self.args.select: diff --git a/core/dbt/task/retry.py b/core/dbt/task/retry.py index 3a14932aea8..d835bed1d2e 100644 --- a/core/dbt/task/retry.py +++ b/core/dbt/task/retry.py @@ -46,7 +46,7 @@ class RetryTask(ConfiguredTask): - def __init__(self, args, config, manifest): + def __init__(self, args, config, manifest) -> None: super().__init__(args, config, manifest) state_path = self.args.state or self.config.target_path @@ -67,7 +67,7 @@ def __init__(self, args, config, manifest): self.previous_args = self.previous_state.results.args self.previous_command_name = self.previous_args.get("which") - self.task_class = TASK_DICT.get(self.previous_command_name) + self.task_class = TASK_DICT.get(self.previous_command_name) # type: ignore def run(self): unique_ids = set( diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index 5c2f2d1f094..a046c4b22e1 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -52,7 +52,7 @@ class Timer: - def __init__(self): + def __init__(self) -> None: self.start = None self.end = None diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index cce7793b598..561b97bdef4 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -5,7 +5,7 @@ from datetime import datetime from multiprocessing.dummy import Pool as ThreadPool from pathlib import Path -from typing import Optional, Dict, List, Set, Tuple, Iterable, AbstractSet +from typing import AbstractSet, Optional, Dict, List, Set, Tuple, Iterable import dbt.exceptions import dbt.tracking diff --git a/core/dbt/task/show.py b/core/dbt/task/show.py index f9af847e874..ac47615effc 100644 --- a/core/dbt/task/show.py +++ b/core/dbt/task/show.py @@ -13,7 +13,7 @@ class ShowRunner(CompileRunner): - def __init__(self, config, adapter, node, node_index, num_nodes): + def __init__(self, config, adapter, node, node_index, num_nodes) -> None: super().__init__(config, adapter, node, node_index, num_nodes) self.run_ephemeral_models = True diff --git a/core/dbt/task/sql.py b/core/dbt/task/sql.py index 4f662383d74..2e84f94a4e8 100644 --- a/core/dbt/task/sql.py +++ b/core/dbt/task/sql.py @@ -19,7 +19,7 @@ class GenericSqlRunner(CompileRunner, Generic[SQLResult]): - def __init__(self, config, adapter, node, node_index, num_nodes): + def __init__(self, config, adapter, node, node_index, num_nodes) -> None: CompileRunner.__init__(self, config, adapter, node, node_index, num_nodes) def handle_exception(self, e, ctx): diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index 0f5e4ca99d0..c0af8baa5df 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -183,7 +183,7 @@ def after_execute(self, result): class TestSelector(ResourceTypeSelector): - def __init__(self, graph, manifest, previous_state): + def __init__(self, graph, manifest, previous_state) -> None: super().__init__( graph=graph, manifest=manifest, diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index f7c0a2a7d19..88022c93f0f 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -50,7 +50,7 @@ class TimeoutEmitter(Emitter): - def __init__(self): + def __init__(self) -> None: super().__init__( COLLECTOR_URL, protocol=COLLECTOR_PROTOCOL, @@ -110,7 +110,7 @@ def http_get(self, payload): class User: - def __init__(self, cookie_dir): + def __init__(self, cookie_dir) -> None: self.do_not_track = True self.cookie_dir = cookie_dir @@ -457,7 +457,7 @@ def do_not_track(): class InvocationProcessor(logbook.Processor): - def __init__(self): + def __init__(self) -> None: super().__init__() def process(self, record): diff --git a/core/dbt/utils.py b/core/dbt/utils.py index e517c2bd304..2386d226aab 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -239,7 +239,7 @@ def deep_map_render(func: Callable[[Any, Tuple[Union[str, int], ...]], Any], val class AttrDict(dict): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.__dict__ = self @@ -284,9 +284,9 @@ class memoized: Taken from https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize""" - def __init__(self, func): + def __init__(self, func) -> None: self.func = func - self.cache = {} + self.cache: Dict[Any, Any] = {} def __call__(self, *args): if not isinstance(args, collections.abc.Hashable): @@ -367,7 +367,7 @@ def default(self, obj): class Translator: - def __init__(self, aliases: Mapping[str, str], recursive: bool = False): + def __init__(self, aliases: Mapping[str, str], recursive: bool = False) -> None: self.aliases = aliases self.recursive = recursive @@ -457,7 +457,7 @@ def lowercase(value: Optional[str]) -> Optional[str]: class classproperty(object): - def __init__(self, func): + def __init__(self, func) -> None: self.func = func def __get__(self, obj, objtype): diff --git a/core/setup.py b/core/setup.py index 5179e5a8fd1..3929e4dd82c 100644 --- a/core/setup.py +++ b/core/setup.py @@ -59,7 +59,7 @@ # ---- # dbt-core uses these packages in standard ways. Pin to the major version, and check compatibility # with major versions in each new minor version of dbt-core. - "click<9", + "click>=8.0.2,<9", "networkx>=2.3,<4", # ---- # These packages are major-version-0. Keep upper bounds on upcoming minor versions (which could have breaking changes) diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py new file mode 100644 index 00000000000..3e987aebb28 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_date_spine.py @@ -0,0 +1,92 @@ +# If date_spine works properly, there should be no `null` values in the resulting model + +models__test_date_spine_sql = """ +with generated_dates as ( + {% if target.type == 'postgres' %} + {{ date_spine("day", "'2023-09-01'::date", "'2023-09-10'::date") }} + + {% elif target.type == 'bigquery' or target.type == 'redshift' %} + select cast(date_day as date) as date_day + from ({{ date_spine("day", "'2023-09-01'", "'2023-09-10'") }}) + + {% else %} + {{ date_spine("day", "'2023-09-01'", "'2023-09-10'") }} + {% endif %} +), expected_dates as ( + {% if target.type == 'postgres' %} + select '2023-09-01'::date as expected + union all + select '2023-09-02'::date as expected + union all + select '2023-09-03'::date as expected + union all + select '2023-09-04'::date as expected + union all + select '2023-09-05'::date as expected + union all + select '2023-09-06'::date as expected + union all + select '2023-09-07'::date as expected + union all + select '2023-09-08'::date as expected + union all + select '2023-09-09'::date as expected + + {% elif target.type == 'bigquery' or target.type == 'redshift' %} + select cast('2023-09-01' as date) as expected + union all + select cast('2023-09-02' as date) as expected + union all + select cast('2023-09-03' as date) as expected + union all + select cast('2023-09-04' as date) as expected + union all + select cast('2023-09-05' as date) as expected + union all + select cast('2023-09-06' as date) as expected + union all + select cast('2023-09-07' as date) as expected + union all + select cast('2023-09-08' as date) as expected + union all + select cast('2023-09-09' as date) as expected + + {% else %} + select '2023-09-01' as expected + union all + select '2023-09-02' as expected + union all + select '2023-09-03' as expected + union all + select '2023-09-04' as expected + union all + select '2023-09-05' as expected + union all + select '2023-09-06' as expected + union all + select '2023-09-07' as expected + union all + select '2023-09-08' as expected + union all + select '2023-09-09' as expected + {% endif %} +), joined as ( + select + generated_dates.date_day, + expected_dates.expected + from generated_dates + left join expected_dates on generated_dates.date_day = expected_dates.expected +) + +SELECT * from joined +""" + +models__test_date_spine_yml = """ +version: 2 +models: + - name: test_date_spine + tests: + - assert_equal: + actual: date_day + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py b/tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py new file mode 100644 index 00000000000..0ce578df9b2 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_generate_series.py @@ -0,0 +1,45 @@ +# If generate_series works properly, there should be no `null` values in the resulting model + +models__test_generate_series_sql = """ +with generated_numbers as ( + {{ dbt.generate_series(10) }} +), expected_numbers as ( + select 1 as expected + union all + select 2 as expected + union all + select 3 as expected + union all + select 4 as expected + union all + select 5 as expected + union all + select 6 as expected + union all + select 7 as expected + union all + select 8 as expected + union all + select 9 as expected + union all + select 10 as expected +), joined as ( + select + generated_numbers.generated_number, + expected_numbers.expected + from generated_numbers + left join expected_numbers on generated_numbers.generated_number = expected_numbers.expected +) + +SELECT * from joined +""" + +models__test_generate_series_yml = """ +version: 2 +models: + - name: test_generate_series + tests: + - assert_equal: + actual: generated_number + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py b/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py new file mode 100644 index 00000000000..bd1b0ddea4b --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_get_intervals_between.py @@ -0,0 +1,20 @@ +models__test_get_intervals_between_sql = """ +SELECT + {% if target.type == 'postgres' %} + {{ get_intervals_between("'09/01/2023'::date", "'09/12/2023'::date", "day") }} as intervals, + {% else %} + {{ get_intervals_between("'09/01/2023'", "'09/12/2023'", "day") }} as intervals, + {% endif %} + 11 as expected + +""" + +models__test_get_intervals_between_yml = """ +version: 2 +models: + - name: test_get_intervals_between + tests: + - assert_equal: + actual: intervals + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py b/tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py new file mode 100644 index 00000000000..04ace2062f6 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/fixture_get_powers_of_two.py @@ -0,0 +1,39 @@ +# get_powers_of_two + +models__test_get_powers_of_two_sql = """ +select {{ get_powers_of_two(1) }} as actual, 1 as expected + +union all + +select {{ get_powers_of_two(4) }} as actual, 2 as expected + +union all + +select {{ get_powers_of_two(27) }} as actual, 5 as expected + +union all + +select {{ get_powers_of_two(256) }} as actual, 8 as expected + +union all + +select {{ get_powers_of_two(3125) }} as actual, 12 as expected + +union all + +select {{ get_powers_of_two(46656) }} as actual, 16 as expected + +union all + +select {{ get_powers_of_two(823543) }} as actual, 20 as expected +""" + +models__test_get_powers_of_two_yml = """ +version: 2 +models: + - name: test_powers_of_two + tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/tests/adapter/dbt/tests/adapter/utils/test_date_spine.py b/tests/adapter/dbt/tests/adapter/utils/test_date_spine.py new file mode 100644 index 00000000000..0a5d7b7d29f --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_date_spine.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_date_spine import ( + models__test_date_spine_sql, + models__test_date_spine_yml, +) + + +class BaseDateSpine(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_date_spine.yml": models__test_date_spine_yml, + "test_date_spine.sql": self.interpolate_macro_namespace( + models__test_date_spine_sql, "date_spine" + ), + } + + +class TestDateSpine(BaseDateSpine): + pass diff --git a/tests/adapter/dbt/tests/adapter/utils/test_generate_series.py b/tests/adapter/dbt/tests/adapter/utils/test_generate_series.py new file mode 100644 index 00000000000..afc8d77dd3b --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_generate_series.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_generate_series import ( + models__test_generate_series_sql, + models__test_generate_series_yml, +) + + +class BaseGenerateSeries(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_generate_series.yml": models__test_generate_series_yml, + "test_generate_series.sql": self.interpolate_macro_namespace( + models__test_generate_series_sql, "generate_series" + ), + } + + +class TestGenerateSeries(BaseGenerateSeries): + pass diff --git a/tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py b/tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py new file mode 100644 index 00000000000..588d2c538d7 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_get_intervals_between.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_get_intervals_between import ( + models__test_get_intervals_between_sql, + models__test_get_intervals_between_yml, +) + + +class BaseGetIntervalsBetween(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_get_intervals_between.yml": models__test_get_intervals_between_yml, + "test_get_intervals_between.sql": self.interpolate_macro_namespace( + models__test_get_intervals_between_sql, "get_intervals_between" + ), + } + + +class TestGetIntervalsBetween(BaseGetIntervalsBetween): + pass diff --git a/tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py b/tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py new file mode 100644 index 00000000000..aa6f4d1a196 --- /dev/null +++ b/tests/adapter/dbt/tests/adapter/utils/test_get_powers_of_two.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.utils.base_utils import BaseUtils +from dbt.tests.adapter.utils.fixture_get_powers_of_two import ( + models__test_get_powers_of_two_sql, + models__test_get_powers_of_two_yml, +) + + +class BaseGetPowersOfTwo(BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_get_powers_of_two.yml": models__test_get_powers_of_two_yml, + "test_get_powers_of_two.sql": self.interpolate_macro_namespace( + models__test_get_powers_of_two_sql, "get_powers_of_two" + ), + } + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass diff --git a/tests/functional/cli/test_multioption.py b/tests/functional/cli/test_multioption.py new file mode 100644 index 00000000000..e9013fdb658 --- /dev/null +++ b/tests/functional/cli/test_multioption.py @@ -0,0 +1,142 @@ +import pytest +from dbt.tests.util import run_dbt + + +model_one_sql = """ +select 1 as fun +""" + +schema_sql = """ +sources: + - name: my_source + description: "My source" + schema: test_schema + tables: + - name: my_table + - name: my_other_table + +exposures: + - name: weekly_jaffle_metrics + label: By the Week + type: dashboard + maturity: high + url: https://bi.tool/dashboards/1 + description: > + Did someone say "exponential growth"? + depends_on: + - ref('model_one') + owner: + name: dbt Labs + email: data@jaffleshop.com +""" + + +class TestResourceType: + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": schema_sql, "model_one.sql": model_one_sql} + + def test_resource_type_single(self, project): + result = run_dbt(["-q", "ls", "--resource-types", "model"]) + assert len(result) == 1 + assert result == ["test.model_one"] + + def test_resource_type_quoted(self, project): + result = run_dbt(["-q", "ls", "--resource-types", "model source"]) + assert len(result) == 3 + expected_result = { + "test.model_one", + "source:test.my_source.my_table", + "source:test.my_source.my_other_table", + } + assert set(result) == expected_result + + def test_resource_type_args(self, project): + result = run_dbt( + [ + "-q", + "ls", + "--resource-type", + "model", + "--resource-type", + "source", + "--resource-type", + "exposure", + ] + ) + assert len(result) == 4 + expected_result = { + "test.model_one", + "source:test.my_source.my_table", + "source:test.my_source.my_other_table", + "exposure:test.weekly_jaffle_metrics", + } + assert set(result) == expected_result + + +class TestOutputKeys: + @pytest.fixture(scope="class") + def models(self): + return {"model_one.sql": model_one_sql} + + def test_output_key_single(self, project): + result = run_dbt(["-q", "ls", "--output", "json", "--output-keys", "name"]) + assert len(result) == 1 + assert result == ['{"name": "model_one"}'] + + def test_output_key_quoted(self, project): + result = run_dbt(["-q", "ls", "--output", "json", "--output-keys", "name resource_type"]) + + assert len(result) == 1 + assert result == ['{"name": "model_one", "resource_type": "model"}'] + + def test_output_key_args(self, project): + result = run_dbt( + [ + "-q", + "ls", + "--output", + "json", + "--output-keys", + "name", + "--output-keys", + "resource_type", + ] + ) + + assert len(result) == 1 + assert result == ['{"name": "model_one", "resource_type": "model"}'] + + +class TestSelectExclude: + @pytest.fixture(scope="class") + def models(self): + return { + "model_one.sql": model_one_sql, + "model_two.sql": model_one_sql, + "model_three.sql": model_one_sql, + } + + def test_select_exclude_single(self, project): + result = run_dbt(["-q", "ls", "--select", "model_one"]) + assert len(result) == 1 + assert result == ["test.model_one"] + result = run_dbt(["-q", "ls", "--exclude", "model_one"]) + assert len(result) == 2 + assert "test.model_one" not in result + + def test_select_exclude_quoted(self, project): + result = run_dbt(["-q", "ls", "--select", "model_one model_two"]) + assert len(result) == 2 + assert "test.model_three" not in result + result = run_dbt(["-q", "ls", "--exclude", "model_one model_two"]) + assert len(result) == 1 + assert result == ["test.model_three"] + + def test_select_exclude_args(self, project): + result = run_dbt(["-q", "ls", "--select", "model_one", "--select", "model_two"]) + assert len(result) == 2 + assert "test.model_three" not in result + result = run_dbt(["-q", "ls", "--exclude", "model_one", "--exclude", "model_two"]) + assert len(result) == 1 + assert result == ["test.model_three"] diff --git a/tests/functional/metrics/fixtures.py b/tests/functional/metrics/fixtures.py index 65d61ad74ad..89eeb930043 100644 --- a/tests/functional/metrics/fixtures.py +++ b/tests/functional/metrics/fixtures.py @@ -116,6 +116,8 @@ measure: name: years_tenure filter: "{{ Dimension('id__loves_dbt') }} is true" + join_to_timespine: true + fill_nulls_with: 0 - name: collective_window label: "Collective window" diff --git a/tests/functional/retry/fixtures.py b/tests/functional/retry/fixtures.py index 1c063b4490a..caa1191b837 100644 --- a/tests/functional/retry/fixtures.py +++ b/tests/functional/retry/fixtures.py @@ -45,3 +45,16 @@ {% do log("Timezone set to: " + timezone, info=True) %} {% endmacro %} """ + +simple_model = """ +select null as id +""" + +simple_schema = """ +models: + - name: some_model + columns: + - name: id + tests: + - not_null +""" diff --git a/tests/functional/retry/test_retry.py b/tests/functional/retry/test_retry.py index 8c322a664c7..c028bc33f45 100644 --- a/tests/functional/retry/test_retry.py +++ b/tests/functional/retry/test_retry.py @@ -9,6 +9,8 @@ schema_yml, models__second_model, macros__alter_timezone_sql, + simple_model, + simple_schema, ) @@ -225,3 +227,47 @@ def test_fail_fast(self, project): results = run_dbt(["retry"]) assert {r.node.unique_id: r.status for r in results.results} == {} + + +class TestRetryResourceType: + @pytest.fixture(scope="class") + def models(self): + return { + "null_model.sql": simple_model, + "schema.yml": simple_schema, + } + + def test_resource_type(self, project): + # test multiple options in single string + results = run_dbt(["build", "--select", "null_model", "--resource-type", "test model"]) + assert len(results) == 1 + + # nothing to do + results = run_dbt(["retry"]) + assert len(results) == 0 + + # test multiple options in multiple args + results = run_dbt( + [ + "build", + "--select", + "null_model", + "--resource-type", + "test", + "--resource-type", + "model", + ] + ) + assert len(results) == 1 + + # nothing to do + results = run_dbt(["retry"]) + assert len(results) == 0 + + # test single all option + results = run_dbt(["build", "--select", "null_model", "--resource-type", "all"]) + assert len(results) == 1 + + # nothing to do + results = run_dbt(["retry"]) + assert len(results) == 0 diff --git a/tests/functional/semantic_models/fixtures.py b/tests/functional/semantic_models/fixtures.py index 12cb3d9d68b..d19ba42f1a6 100644 --- a/tests/functional/semantic_models/fixtures.py +++ b/tests/functional/semantic_models/fixtures.py @@ -58,23 +58,29 @@ semantic_models: - name: semantic_people + label: "Semantic People" model: ref('people') dimensions: - name: favorite_color + label: "Favorite Color" type: categorical - name: created_at + label: "Created At" type: TIME type_params: time_granularity: day measures: - name: years_tenure + label: "Years Tenure" agg: SUM expr: tenure - name: people + label: "People" agg: count expr: id entities: - name: id + label: "Primary ID" type: primary defaults: agg_time_dimension: created_at @@ -85,6 +91,7 @@ semantic_models: - name: semantic_people + label: "Semantic People" model: ref('people') config: enabled: true @@ -115,6 +122,7 @@ semantic_models: - name: semantic_people + label: "Semantic People" model: ref('people') config: enabled: false diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index e4d347ad4d0..83c0e251deb 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -383,7 +383,7 @@ def test_from_dict__run(self): } result = self._create_flags_from_dict(Command.RUN, args_dict) assert "model_one" in result.select[0] - assert "model_two" in result.select[0] + assert "model_two" in result.select[1] def test_from_dict__build(self): args_dict = { diff --git a/tests/unit/test_contracts_graph_unparsed.py b/tests/unit/test_contracts_graph_unparsed.py index 7f29937a007..fbf23a082c7 100644 --- a/tests/unit/test_contracts_graph_unparsed.py +++ b/tests/unit/test_contracts_graph_unparsed.py @@ -858,6 +858,7 @@ def get_ok_dict(self): "measure": { "name": "customers", "filter": "is_new = true", + "join_to_timespine": False, }, }, "config": {}, diff --git a/tests/unit/test_event_handler.py b/tests/unit/test_event_handler.py new file mode 100644 index 00000000000..905b23f5a11 --- /dev/null +++ b/tests/unit/test_event_handler.py @@ -0,0 +1,40 @@ +import logging + +from dbt.events.base_types import EventLevel +from dbt.events.event_handler import DbtEventLoggingHandler, set_package_logging +from dbt.events.eventmgr import TestEventManager + + +def test_event_logging_handler_emits_records_correctly(): + event_manager = TestEventManager() + handler = DbtEventLoggingHandler(event_manager=event_manager, level=logging.DEBUG) + log = logging.getLogger("test") + log.setLevel(logging.DEBUG) + log.addHandler(handler) + + log.debug("test") + log.info("test") + log.warning("test") + log.error("test") + log.exception("test") + log.critical("test") + assert len(event_manager.event_history) == 6 + assert event_manager.event_history[0][1] == EventLevel.DEBUG + assert event_manager.event_history[1][1] == EventLevel.INFO + assert event_manager.event_history[2][1] == EventLevel.WARN + assert event_manager.event_history[3][1] == EventLevel.ERROR + assert event_manager.event_history[4][1] == EventLevel.ERROR + assert event_manager.event_history[5][1] == EventLevel.ERROR + + +def test_set_package_logging_sets_level_correctly(): + event_manager = TestEventManager() + log = logging.getLogger("test") + set_package_logging("test", logging.DEBUG, event_manager) + log.debug("debug") + assert len(event_manager.event_history) == 1 + set_package_logging("test", logging.WARN, event_manager) + log.debug("debug 2") + assert len(event_manager.event_history) == 1 + log.warning("warning") + assert len(event_manager.event_history) == 3 # warning logs create two events diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index 8c93381c938..64e24429965 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -1,3 +1,4 @@ +import logging import re from argparse import Namespace from typing import TypeVar @@ -83,6 +84,12 @@ def test_formatting(self): event = types.JinjaLogDebug(msg=[1, 2, 3]) assert isinstance(event.msg, str) + def test_set_adapter_dependency_log_level(self): + logger = AdapterLogger("dbt_tests") + package_log = logging.getLogger("test_package_log") + logger.set_adapter_dependency_log_level("test_package_log", "DEBUG") + package_log.debug("debug message") + class TestEventCodes: diff --git a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py index 948540f4805..85d24797ef1 100644 --- a/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py +++ b/tests/unit/test_semantic_layer_nodes_satisfy_protocols.py @@ -216,7 +216,11 @@ def simple_metric_input_measure() -> MetricInputMeasure: @pytest.fixture(scope="session") def complex_metric_input_measure(where_filter) -> MetricInputMeasure: return MetricInputMeasure( - name="test_complex_metric_input_measure", filter=where_filter, alias="complex_alias" + name="test_complex_metric_input_measure", + filter=where_filter, + alias="complex_alias", + join_to_timespine=True, + fill_nulls_with=0, ) @@ -312,6 +316,7 @@ def test_semantic_model_node_satisfies_protocol_optionals_specified( schema_name="test_schema_name", ), description="test_description", + label="test label", defaults=semantic_model_defaults, metadata=source_file_metadata, primary_entity="test_primary_entity", @@ -334,6 +339,7 @@ def test_dimension_satisfies_protocol_optionals_specified( name="test_dimension", type=DimensionType.TIME, description="test_description", + label="test_label", type_params=dimension_type_params, expr="1", metadata=source_file_metadata, @@ -353,6 +359,7 @@ def test_entity_satisfies_protocol_optionals_specified(): entity = Entity( name="test_entity", description="a test entity", + label="A test entity", type=EntityType.PRIMARY, expr="id", role="a_role", @@ -374,6 +381,7 @@ def test_measure_satisfies_protocol_optionals_specified( measure = Measure( name="test_measure", description="a test measure", + label="A test measure", agg="sum", create_metric=True, expr="amount",